Compare commits
56
Commits
+2
-3
@@ -56,9 +56,8 @@ SECRET_KEY=
|
||||
# LINE. Leaving it set disarms the protection permanently, on an instance that
|
||||
# by then has credentials worth protecting.
|
||||
#
|
||||
# BACK UP ./images/secrets/ ALONGSIDE YOUR DATABASE. The key is the only thing
|
||||
# that can read your stored credentials; a database restored without it needs
|
||||
# every credential re-entered by hand.
|
||||
# Backups leave the key and the stored logins out on purpose: after restoring
|
||||
# onto a new install, sign in to each account again.
|
||||
CURATOR_BOOTSTRAP_NEW_KEY=1
|
||||
|
||||
|
||||
|
||||
@@ -81,8 +81,18 @@ jobs:
|
||||
set -eux
|
||||
# Same service-IP dance as build.yml's integration job; see the long
|
||||
# comment there for why the job name must stay separator-free.
|
||||
PG=$(docker ps --filter "name=compare" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
test -n "$PG"
|
||||
# Only THIS job's services. The runner's docker daemon is shared, so another
|
||||
# repo's job can be running beside this one, and a job-name filter matches its
|
||||
# containers too (Steward run 8358 and Inkwell run 8653 each took another repo's
|
||||
# Postgres). act names every container of a task GITEA-ACTIONS-TASK-<n>-...: read
|
||||
# <n> from our own container (its id is in the /etc/hostname bind mount) and
|
||||
# require exactly one match.
|
||||
SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true)
|
||||
SELF=${SELF:-$(hostname)}
|
||||
TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-')
|
||||
test -n "$TASK"
|
||||
PG=$(docker ps --filter "name=$TASK" --filter "ancestor=pgvector/pgvector:pg16" -q)
|
||||
test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
test -n "$PG_IP"
|
||||
echo "PG_CONTAINER=$PG" >> "$GITHUB_ENV"
|
||||
|
||||
+203
-18
@@ -142,6 +142,16 @@ concurrency:
|
||||
# Deriving it per job invites the two halves to disagree: sign-extension would
|
||||
# derive dev's extension version while build-web bundled main's, and the
|
||||
# release download would 404 on a version that exists perfectly well.
|
||||
#
|
||||
# On every other trigger it is the COMMIT that fired (`github.sha`), never the
|
||||
# branch name. A branch is re-resolved by each job's checkout when that job
|
||||
# starts, so a push landing mid-run moved the later jobs onto the new tip: run
|
||||
# 7499 signed 423275a's extension, then build-web checked out 83e1382 (pushed
|
||||
# while 7499 ran), derived a version nobody had signed, and 404'd on the
|
||||
# download (#4427). The guard failed closed that time; a job without one would
|
||||
# have published a commit the run's own lanes never tested — the lanes check
|
||||
# out `github.sha` by default, so pinning here makes the publish build exactly
|
||||
# what they passed.
|
||||
# IS THIS A BASE REFRESH? Asked in five places and previously spelled five
|
||||
# ways — `github.event_name == 'schedule'` in an `if:`, `$GITHUB_EVENT_NAME` in
|
||||
# one shell, an `EVENT:` env passed into another, and a bare expression on
|
||||
@@ -178,12 +188,22 @@ concurrency:
|
||||
# where a step-level `if:` needs the answer before any shell runs.
|
||||
env:
|
||||
IS_REFRESH: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'true' || 'false' }}
|
||||
BUILD_REF: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'main' || github.ref }}
|
||||
BUILD_REF: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'main' || github.sha }}
|
||||
# What the six LANES check out. Empty — the checkout default, the triggering
|
||||
# commit (or a PR's merge ref) — on every trigger but the refresh, where it is
|
||||
# `main`: the refresh publishes main, so the gate has to test main (#4430). It
|
||||
# is not BUILD_REF itself because a pull_request run's `github.sha` is a merge
|
||||
# commit the default checkout reaches through its ref, not by sha. Each job
|
||||
# still resolves `main` when it starts, so a merge to main during the ~5 min of
|
||||
# a Sunday-06:00 refresh could put the lanes and the build one commit apart;
|
||||
# the build jobs' own guards assert the branch, not the commit.
|
||||
LANE_REF: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'main' || '' }}
|
||||
|
||||
# Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
|
||||
# - write:package, read:package (for docker push to git.fabledsword.com)
|
||||
# - write:release (for ext-<version> release asset cache)
|
||||
# - write:issue (for future issue-management automation)
|
||||
# - write:issue (`unattended-failure` opens an issue)
|
||||
# - write:repository (`refresh-dev` dispatches this workflow)
|
||||
# The injected GITHUB_TOKEN cannot be used — it lacks write:package.
|
||||
|
||||
jobs:
|
||||
@@ -219,6 +239,8 @@ jobs:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ env.LANE_REF }}
|
||||
- name: Ruff lint
|
||||
# agent/ included so the GPU-agent is linted before its image is built
|
||||
# (build.yml only `docker build`s it — this is where it gets checked).
|
||||
@@ -265,6 +287,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ env.LANE_REF }}
|
||||
# The derivation needs real history: a depth-1 clone sees one commit
|
||||
# and produces a wrong, too-low value RATHER THAN FAILING. Checking
|
||||
# that here is half the point of the lane.
|
||||
@@ -320,6 +343,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ env.LANE_REF }}
|
||||
# Full history for tests/test_artifact_identity.py, which derives
|
||||
# each artifact's revision to check the identity scheme. On a
|
||||
# depth-1 clone that derivation either fails or returns the tip sha
|
||||
@@ -366,6 +390,8 @@ jobs:
|
||||
working-directory: frontend
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ env.LANE_REF }}
|
||||
# No package-lock.json is tracked yet (we don't run npm locally per
|
||||
# feedback-no-local-runs). Using `npm install` instead of `npm ci`.
|
||||
# If we want strict lockfile-based reproducibility later, commit a
|
||||
@@ -395,6 +421,8 @@ jobs:
|
||||
image: node:24-bookworm-slim
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ env.LANE_REF }}
|
||||
# Not --no-save: vitest and web-ext are both real devDependencies now,
|
||||
# and the suite needs vitest resolvable from node_modules.
|
||||
- name: Install dev dependencies
|
||||
@@ -497,15 +525,28 @@ jobs:
|
||||
--health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ env.LANE_REF }}
|
||||
- name: Integration suite (resolve service IPs, migrate, test)
|
||||
run: |
|
||||
set -eux
|
||||
echo "=== container landscape (diagnostic for filter scoping) ==="
|
||||
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
|
||||
echo "=== end landscape ==="
|
||||
PG=$(docker ps --filter "name=integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=integration" --filter "ancestor=redis:8-alpine" -q | head -n1)
|
||||
test -n "$PG" && test -n "$RD"
|
||||
# Only THIS job's services. The runner's docker daemon is shared, so another
|
||||
# repo's job can be running beside this one, and a job-name filter matches its
|
||||
# containers too (Steward run 8358 and Inkwell run 8653 each took another repo's
|
||||
# Postgres). act names every container of a task GITEA-ACTIONS-TASK-<n>-...: read
|
||||
# <n> from our own container (its id is in the /etc/hostname bind mount) and
|
||||
# require exactly one match.
|
||||
SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true)
|
||||
SELF=${SELF:-$(hostname)}
|
||||
TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-')
|
||||
test -n "$TASK"
|
||||
PG=$(docker ps --filter "name=$TASK" --filter "ancestor=pgvector/pgvector:pg16" -q)
|
||||
test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1
|
||||
RD=$(docker ps --filter "name=$TASK" --filter "ancestor=redis:8-alpine" -q)
|
||||
test "$(printf '%s\n' "$RD" | grep -c .)" -eq 1
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||
test -n "$PG_IP" && test -n "$RD_IP"
|
||||
@@ -538,6 +579,9 @@ jobs:
|
||||
else
|
||||
pip install -r requirements.txt pytest pytest-asyncio
|
||||
fi
|
||||
# The backup round trip runs the real pg_dump and pg_restore, from the
|
||||
# same Debian client package the app image installs.
|
||||
command -v pg_dump >/dev/null 2>&1 || { apt-get update -qq && apt-get install -y -qq postgresql-client; }
|
||||
# Relax durability on the throwaway CI Postgres so the per-test
|
||||
# TRUNCATE's commit-fsync — the integration teardown's dominant cost
|
||||
# (~1.5-2s/test, which collapsed the suite from ~13min to ~45s) — is
|
||||
@@ -605,8 +649,8 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Not the triggering ref — see the `env:` block at the top. On a
|
||||
# scheduled refresh this is `main`; on everything else it is the ref
|
||||
# that fired, so this is a no-op on every ordinary path.
|
||||
# scheduled refresh this is `main`; on everything else it is the
|
||||
# commit that fired, the same one the lanes above tested.
|
||||
ref: ${{ env.BUILD_REF }}
|
||||
# Full history is load-bearing, not a convenience: the version this
|
||||
# job signs is derived from the commit TIME of the newest packaged
|
||||
@@ -747,6 +791,15 @@ jobs:
|
||||
print("guard: no ext-* release yet — nothing to go backwards from")
|
||||
raise SystemExit(0)
|
||||
|
||||
# A version that is already signed is reused from its release, not
|
||||
# signed again, so it cannot be a downgrade. The weekly refresh builds
|
||||
# main, whose version trails whatever dev has signed since; refusing
|
||||
# it here would stop :latest from ever being refreshed while dev is
|
||||
# ahead.
|
||||
if any(v == derived for v, _ in signed):
|
||||
print(f"guard: {derived_s} is already signed — reused, not re-signed")
|
||||
raise SystemExit(0)
|
||||
|
||||
hi, hi_tag = signed[-1]
|
||||
print(f"guard: derived={derived_s} highest already signed={hi_tag}")
|
||||
if derived < hi:
|
||||
@@ -945,8 +998,8 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Not the triggering ref — see the `env:` block at the top. On a
|
||||
# scheduled refresh this is `main`; on everything else it is the ref
|
||||
# that fired, so this is a no-op on every ordinary path.
|
||||
# scheduled refresh this is `main`; on everything else it is the
|
||||
# commit that fired, the same one the lanes above tested.
|
||||
ref: ${{ env.BUILD_REF }}
|
||||
# Full history: this job RE-DERIVES the extension version rather than
|
||||
# being handed it, and a depth-1 clone derives a wrong, too-low value
|
||||
@@ -1397,8 +1450,11 @@ jobs:
|
||||
# `no-cache: true`, which is a much larger version of the same
|
||||
# churn #3265 is about.
|
||||
#
|
||||
# Only on the schedule. An ordinary push wants the cached base.
|
||||
pull: ${{ env.IS_REFRESH == 'true' }}
|
||||
# Only on the refresh and on a forced build. An ordinary push wants
|
||||
# the cached base. A forced build pulls because forcing is how the
|
||||
# refresh rebuilds `:dev` (see `refresh-dev`), and a rebuild that
|
||||
# keeps the old base layers would be the same image again.
|
||||
pull: ${{ env.IS_REFRESH == 'true' || format('{0}', github.event.inputs.force_build) == 'true' }}
|
||||
# ONE tag, the channel's. Every other tag is written by the step
|
||||
# below, registry-side. buildx here pushes the first tag to the
|
||||
# registry and then re-pushes the rest through the DOCKER driver,
|
||||
@@ -1689,9 +1745,20 @@ jobs:
|
||||
# Service discovery mirrors the integration lane above: these jobs run
|
||||
# in a container against a mounted docker socket, so the services are
|
||||
# SIBLINGS reachable by IP, not by hostname.
|
||||
PG=$(docker ps --filter "name=smoke" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=smoke" --filter "ancestor=redis:8-alpine" -q | head -n1)
|
||||
test -n "$PG" && test -n "$RD"
|
||||
# Only THIS job's services. The runner's docker daemon is shared, so another
|
||||
# repo's job can be running beside this one, and a job-name filter matches its
|
||||
# containers too (Steward run 8358 and Inkwell run 8653 each took another repo's
|
||||
# Postgres). act names every container of a task GITEA-ACTIONS-TASK-<n>-...: read
|
||||
# <n> from our own container (its id is in the /etc/hostname bind mount) and
|
||||
# require exactly one match.
|
||||
SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true)
|
||||
SELF=${SELF:-$(hostname)}
|
||||
TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-')
|
||||
test -n "$TASK"
|
||||
PG=$(docker ps --filter "name=$TASK" --filter "ancestor=pgvector/pgvector:pg16" -q)
|
||||
test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1
|
||||
RD=$(docker ps --filter "name=$TASK" --filter "ancestor=redis:8-alpine" -q)
|
||||
test "$(printf '%s\n' "$RD" | grep -c .)" -eq 1
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||
test -n "$PG_IP" && test -n "$RD_IP"
|
||||
@@ -2142,6 +2209,121 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
echo "promote: $MOVED tag(s) written"
|
||||
|
||||
# The refresh rebuilds `:dev` as well as `:latest`.
|
||||
#
|
||||
# The refresh run itself builds main and publishes `:latest`. `:dev` needs a
|
||||
# build of dev's own source with FC_CHANNEL=dev baked in, so copying main's
|
||||
# refreshed bytes onto `:dev` is not the same thing: the image would report
|
||||
# itself as main, and the reuse check would then keep it that way. Instead
|
||||
# this job dispatches an ordinary dev run with `force_build`. That run goes
|
||||
# through the lanes, the smoke and `promote` like any push, and a forced
|
||||
# build pulls fresh bases (see `pull:` in both build jobs).
|
||||
#
|
||||
# Without this, an artifact whose source is quiet on dev (the agent, most
|
||||
# weeks) keeps the base layers of its last build indefinitely, because
|
||||
# reuse skips its build on every push.
|
||||
#
|
||||
# The dispatched run shares this run's concurrency group (both are on the
|
||||
# dev ref), so it queues until this run has finished. It runs whatever
|
||||
# `promote` did: a failed `:latest` refresh is no reason to leave `:dev`
|
||||
# stale, and `unattended-failure` reports the failure either way.
|
||||
refresh-dev:
|
||||
needs: [promote]
|
||||
if: >-
|
||||
!cancelled() && (github.event_name == 'schedule'
|
||||
|| format('{0}', github.event.inputs.refresh) == 'true')
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
steps:
|
||||
- name: Dispatch a forced dev build
|
||||
env:
|
||||
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||
API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
|
||||
run: |
|
||||
set -eu
|
||||
CODE=$(curl -sS --max-time 30 -o response.txt -w '%{http_code}' -X POST \
|
||||
-H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
|
||||
-d '{"ref":"dev","inputs":{"force_build":"true"}}' \
|
||||
"$API/actions/workflows/build.yml/dispatches")
|
||||
echo "refresh-dev: dispatch answered HTTP $CODE"
|
||||
case "$CODE" in
|
||||
2??) echo "refresh-dev: a forced dev run is queued behind this one" ;;
|
||||
*)
|
||||
cat response.txt >&2
|
||||
echo "refresh-dev: the dispatch was refused. A 403 means RELEASE_TOKEN" >&2
|
||||
echo "refresh-dev: lacks write:repository (see the scope list at the top)." >&2
|
||||
exit 1 ;;
|
||||
esac
|
||||
|
||||
# A run nobody is watching reports its own failure.
|
||||
#
|
||||
# A push run is watched by whoever pushed it. The weekly refresh and the dev
|
||||
# run it dispatches are not, and a red run on the Actions page alone goes
|
||||
# unseen. So any failed run that is not a push or a pull request opens an
|
||||
# issue, or comments on the one already open, naming the run and the jobs
|
||||
# that failed. A manual dispatch that fails reports here too; closing the
|
||||
# issue once it is dealt with is what keeps the next failure visible.
|
||||
#
|
||||
# The issue is written with RELEASE_TOKEN, so it is authored by the token's
|
||||
# owner, and the forge does not notify people of their own actions. It
|
||||
# shows in the repository's open issues rather than as an email.
|
||||
unattended-failure:
|
||||
needs: [lint, extension-version, backend-lint-and-test, frontend-build,
|
||||
extension-test, integration, sign-extension, build-web, smoke-web,
|
||||
build-agent, promote, refresh-dev]
|
||||
if: >-
|
||||
failure() && github.event_name != 'push'
|
||||
&& github.event_name != 'pull_request'
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
steps:
|
||||
- name: Open or update the failure issue
|
||||
env:
|
||||
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||
API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
EVENT: ${{ github.event_name }}
|
||||
REF: ${{ github.ref_name }}
|
||||
NEEDS: ${{ toJSON(needs) }}
|
||||
run: |
|
||||
python3 - <<'EOF'
|
||||
import json, os, urllib.parse, urllib.request
|
||||
|
||||
TITLE = "CI: an unattended build.yml run failed"
|
||||
api, token = os.environ["API"], os.environ["TOKEN"]
|
||||
|
||||
def call(method, path, body=None):
|
||||
req = urllib.request.Request(
|
||||
api + path, method=method,
|
||||
data=json.dumps(body).encode() if body is not None else None,
|
||||
headers={"Authorization": f"token {token}",
|
||||
"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return json.load(resp)
|
||||
|
||||
needs = json.loads(os.environ["NEEDS"])
|
||||
failed = sorted(j for j, v in needs.items() if v.get("result") == "failure")
|
||||
text = (f"Run: {os.environ['RUN_URL']}\n"
|
||||
f"Trigger: `{os.environ['EVENT']}` on `{os.environ['REF']}`\n"
|
||||
f"Failed jobs: {', '.join(f'`{j}`' for j in failed) or '(none reported)'}\n")
|
||||
|
||||
q = urllib.parse.urlencode({"state": "open", "type": "issues", "q": TITLE})
|
||||
existing = [i for i in call("GET", f"/issues?{q}") if i["title"] == TITLE]
|
||||
if existing:
|
||||
n = existing[0]["number"]
|
||||
call("POST", f"/issues/{n}/comments", {"body": text})
|
||||
print(f"unattended-failure: commented on open issue #{n}")
|
||||
else:
|
||||
body = ("A build.yml run that nobody watches failed. Read the run's "
|
||||
"failing job logs; close this issue once it is fixed, so the "
|
||||
"next failure opens a new one.\n\n" + text)
|
||||
n = call("POST", "/issues", {"title": TITLE, "body": body})["number"]
|
||||
print(f"unattended-failure: opened issue #{n}")
|
||||
EOF
|
||||
|
||||
build-agent:
|
||||
# THE GATE (2026-09-23). Every lane above must have PASSED before this job
|
||||
# exists at all — so a red suite does not produce an image, let alone push
|
||||
@@ -2174,8 +2356,8 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Not the triggering ref — see the `env:` block at the top. On a
|
||||
# scheduled refresh this is `main`; on everything else it is the ref
|
||||
# that fired, so this is a no-op on every ordinary path.
|
||||
# scheduled refresh this is `main`; on everything else it is the
|
||||
# commit that fired, the same one the lanes above tested.
|
||||
ref: ${{ env.BUILD_REF }}
|
||||
# Full history: this job derives its artifact's version from the
|
||||
# commit its shipped files last changed in (milestone 313). A
|
||||
@@ -2480,8 +2662,11 @@ jobs:
|
||||
# `no-cache: true`, which is a much larger version of the same
|
||||
# churn #3265 is about.
|
||||
#
|
||||
# Only on the schedule. An ordinary push wants the cached base.
|
||||
pull: ${{ env.IS_REFRESH == 'true' }}
|
||||
# Only on the refresh and on a forced build. An ordinary push wants
|
||||
# the cached base. A forced build pulls because forcing is how the
|
||||
# refresh rebuilds `:dev` (see `refresh-dev`), and a rebuild that
|
||||
# keeps the old base layers would be the same image again.
|
||||
pull: ${{ env.IS_REFRESH == 'true' || format('{0}', github.event.inputs.force_build) == 'true' }}
|
||||
# ONE tag, the channel's. Every other tag is written by the step
|
||||
# below, registry-side. buildx here pushes the first tag to the
|
||||
# registry and then re-pushes the rest through the DOCKER driver,
|
||||
|
||||
@@ -22,7 +22,8 @@ through afterwards.
|
||||
- **ML tagging.** Runs image models in-container to suggest tags, group
|
||||
characters, find near-duplicates and power similarity search. Suggestions are
|
||||
reviewable — it proposes, you confirm, and it learns which proposals you keep
|
||||
rejecting.
|
||||
rejecting. It ships switched off: turn it on under Settings → System when
|
||||
you want it, and it fetches its model weights then.
|
||||
- **Deduplication and provenance.** Everything that arrives is hashed and
|
||||
deduplicated by content, metadata sidecars are read wherever the source
|
||||
writes them, and every file keeps a record of where it came from.
|
||||
@@ -113,15 +114,19 @@ that cannot decrypt anything, leaving an instance that looks healthy while every
|
||||
paywalled download fails. Making you say so once, on an empty install, is the
|
||||
price of that not happening silently later.
|
||||
|
||||
**Which means: back up `./images/secrets/` alongside your database.** It is the
|
||||
only thing that can read your stored credentials. A database restored without it
|
||||
needs every credential entered again by hand.
|
||||
**Your account logins are sealed with that key, and no backup carries either.**
|
||||
FabledCurator's backups (below) leave the key and the logins out on purpose, so
|
||||
a stolen backup holds no way into your accounts. After a restore, sign in to
|
||||
each account again through the browser extension.
|
||||
|
||||
A few other things are worth knowing about the first few minutes:
|
||||
|
||||
- **The ML worker downloads its model weights on first boot**, several GB from
|
||||
HuggingFace into `./models`. Until that finishes, tagging is queued rather
|
||||
than broken. It is idempotent — a restart resumes rather than refetches.
|
||||
- **ML tagging starts switched off, and nothing is downloaded at boot.** Give
|
||||
the ML lane a slot under **Settings → System** and it fetches its model
|
||||
weights then — several GB from HuggingFace into `./models`, shown as a job
|
||||
under **Settings → Activity** that you can watch and retry. Until it
|
||||
finishes, tagging is queued rather than broken, and the fetch only takes
|
||||
what is missing, so turning the lane off and on again does not refetch.
|
||||
- **The gallery starts empty**, and that is the expected state. Add a creator
|
||||
under **Subscriptions** and it fills as posts come down.
|
||||
- **If you already have a library on disk**, there is no screen that imports
|
||||
@@ -173,6 +178,52 @@ Run it for a burst of tagging, stop it to get your card back. It deploys from
|
||||
`agent/docker-compose.yml`, not the main stack — see
|
||||
[agent/README.md](agent/README.md).
|
||||
|
||||
## Backing up and restoring
|
||||
|
||||
Backups live under **Settings → Maintenance → Backups**. With nothing else set
|
||||
up, the database is dumped nightly into `./images/_backups`, which is on the
|
||||
same disk as what it protects. Images and attachments are backed up only to an
|
||||
off-site destination.
|
||||
|
||||
To keep a copy off the machine, set up an off-site destination on the same
|
||||
screen: any S3 bucket (Fabled Archives, MinIO, AWS) and a key that may add and
|
||||
read objects. Each night FabledCurator then sends the database as a dated dump
|
||||
under `database/`, and every image and attachment once, named by its
|
||||
fingerprint, under `media/`. It never deletes from the bucket; the bucket's own
|
||||
retention thins old dumps. The first copy of a large library takes hours; a
|
||||
restart along the way carries on where it stopped rather than starting over.
|
||||
|
||||
Once a week the bucket is checked against what was sent, and anything missing
|
||||
goes again in the next run. When the newest database backup is more than a day
|
||||
old or failed, or the check found files missing, the **Backups** row on
|
||||
**Settings → System** and the dot beside the brand turn amber and say why.
|
||||
|
||||
**Test a backup** proves a dump restores without touching your install: the dump
|
||||
is loaded into a separate, temporary database on the same Postgres server,
|
||||
every table is checked against the row counts recorded as the dump was taken,
|
||||
the copy is brought up to this version, and then it is deleted. It needs about
|
||||
as much free space on the database's disk as the database itself, while it
|
||||
runs. A dump taken before counts were recorded is shown beside the live
|
||||
database instead.
|
||||
|
||||
To rebuild after losing the server, start a fresh install (first boot as
|
||||
above), enter the same destination, and under **Restore from the bucket** choose
|
||||
a dump. The database is replaced, then every file is fetched back to its place
|
||||
and checked against its fingerprint as it arrives; progress shows on the same
|
||||
screen. **Fetch missing files** does only the second half, for a library that
|
||||
lost files but kept its database.
|
||||
|
||||
If the web page isn't reachable, the same restore runs from a shell:
|
||||
|
||||
```bash
|
||||
docker compose exec web python -m backend.app.scripts.restore list
|
||||
docker compose exec web python -m backend.app.scripts.restore database --dump latest --yes
|
||||
```
|
||||
|
||||
On an install with no destination saved yet, add `--endpoint`, `--bucket` and
|
||||
`--access-key`, and put the secret key in `FC_BACKUP_SECRET_KEY` (or type it at
|
||||
the prompt).
|
||||
|
||||
## Upgrading
|
||||
|
||||
```bash
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Retire the sketch/doodle WIP title tier — its tags, its review flags, its toggle.
|
||||
|
||||
Milestone 430, #4428. The soft tier (#1474) tagged `wip` on any post titled
|
||||
sketch / doodle / scribble. Measured on the operator's library it was 6,096 of
|
||||
8,876 wip tags, and its conflict audit filled the Gallery's review strip with
|
||||
2,086 cards, because most finished art scores >= 0.5 on some content head. A
|
||||
"sketch" is usually finished work, so the operator retired it (2026-09-25).
|
||||
|
||||
Data:
|
||||
|
||||
* A soft tag the operator stood behind is kept and relabelled `manual`: one they
|
||||
confirmed (tag_positive_confirmation), or one whose review flag they resolved
|
||||
while leaving the tag on (the strip's "Keep tag").
|
||||
* Every other `wip_title_soft` row is deleted.
|
||||
* Unresolved review flags whose tag is no longer on the image are deleted: the
|
||||
question they ask no longer applies. That is the audit's cards, and any older
|
||||
orphan the same way.
|
||||
|
||||
Then `import_settings.wip_soft_title_tagging_enabled` is dropped. The downgrade
|
||||
restores the column only; deleted tags are not recreated.
|
||||
|
||||
Revision ID: 0112
|
||||
Revises: 0111
|
||||
Create Date: 2026-09-25
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0112"
|
||||
down_revision = "0111"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def retire_soft_wip_tags(conn) -> None:
|
||||
"""The data half, on a plain connection, so a test can run it directly."""
|
||||
conn.execute(sa.text("""
|
||||
UPDATE image_tag it SET source = 'manual'
|
||||
WHERE it.source = 'wip_title_soft'
|
||||
AND (
|
||||
EXISTS (
|
||||
SELECT 1 FROM tag_positive_confirmation c
|
||||
WHERE c.image_record_id = it.image_record_id AND c.tag_id = it.tag_id
|
||||
)
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM presentation_review pr
|
||||
WHERE pr.image_record_id = it.image_record_id AND pr.tag_id = it.tag_id
|
||||
AND pr.resolved_at IS NOT NULL
|
||||
)
|
||||
)
|
||||
"""))
|
||||
conn.execute(sa.text("DELETE FROM image_tag WHERE source = 'wip_title_soft'"))
|
||||
conn.execute(sa.text("""
|
||||
DELETE FROM presentation_review pr
|
||||
WHERE pr.resolved_at IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM image_tag it
|
||||
WHERE it.image_record_id = pr.image_record_id AND it.tag_id = pr.tag_id
|
||||
)
|
||||
"""))
|
||||
|
||||
|
||||
def upgrade():
|
||||
retire_soft_wip_tags(op.get_bind())
|
||||
op.drop_column("import_settings", "wip_soft_title_tagging_enabled")
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"wip_soft_title_tagging_enabled",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.text("false"),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Re-date images whose post's date arrived after they were linked.
|
||||
|
||||
#4431. The native ingesters import a post's media before its record, and the
|
||||
date travels in the record (`_post.json`). Every natively downloaded image was
|
||||
therefore linked to an undated post and kept its download time in both gallery
|
||||
date columns, while the post itself was dated correctly. The importer now
|
||||
re-dates a post's images when its record lands; this repairs the images that
|
||||
landed before that.
|
||||
|
||||
Both columns get back the rules the importer keeps:
|
||||
|
||||
* `effective_date` is the primary post's date (left alone when that post has
|
||||
none, as the importer does);
|
||||
* `earliest_post_date` is the earliest dated post the image is linked to.
|
||||
|
||||
Only rows that differ are written. The downgrade does nothing: the old values
|
||||
were download times that no one chose.
|
||||
|
||||
Revision ID: 0113
|
||||
Revises: 0112
|
||||
Create Date: 2026-09-25
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0113"
|
||||
down_revision = "0112"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def redate_images(conn) -> None:
|
||||
"""The data step, on a plain connection, so a test can run it directly."""
|
||||
conn.execute(sa.text("""
|
||||
UPDATE image_record ir SET effective_date = p.post_date
|
||||
FROM post p
|
||||
WHERE p.id = ir.primary_post_id
|
||||
AND p.post_date IS NOT NULL
|
||||
AND ir.effective_date IS DISTINCT FROM p.post_date
|
||||
"""))
|
||||
conn.execute(sa.text("""
|
||||
UPDATE image_record ir SET earliest_post_date = m.earliest
|
||||
FROM (
|
||||
SELECT ip.image_record_id, MIN(p.post_date) AS earliest
|
||||
FROM image_provenance ip JOIN post p ON p.id = ip.post_id
|
||||
WHERE p.post_date IS NOT NULL
|
||||
GROUP BY ip.image_record_id
|
||||
) m
|
||||
WHERE m.image_record_id = ir.id
|
||||
AND ir.earliest_post_date IS DISTINCT FROM m.earliest
|
||||
"""))
|
||||
|
||||
|
||||
def upgrade():
|
||||
redate_images(op.get_bind())
|
||||
|
||||
|
||||
def downgrade():
|
||||
pass
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Fold the undated shell posts a misfiled attachment created into the real post.
|
||||
|
||||
#4435. A non-image file (an archive, a pdf) downloaded for one of an artist's
|
||||
Discord channels was filed under the artist's FIRST Discord source: the
|
||||
attachment path looked the source up by (artist, platform), which takes the
|
||||
lowest id. That created an undated, url-less post there holding only the
|
||||
attachment, and the message's real post record then created the dated post
|
||||
under the right source. The importer now uses the source it was downloading
|
||||
for; this repairs the pairs it left.
|
||||
|
||||
A shell is folded only when all of this holds:
|
||||
|
||||
* it has no date and no url, and nothing synthesized it;
|
||||
* another post of the same artist, platform and external id HAS a date;
|
||||
* no image is linked to the shell (it held an attachment and nothing else).
|
||||
|
||||
Its attachments move to the dated post, dropping any the dated post already
|
||||
has (same sha256, which the per-post unique forbids twice), and the shell is
|
||||
deleted. A shell with no dated twin is left alone: which channel it belongs to
|
||||
is not recorded anywhere but the file name. The downgrade does nothing.
|
||||
|
||||
Revision ID: 0114
|
||||
Revises: 0113
|
||||
Create Date: 2026-09-25
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0114"
|
||||
down_revision = "0113"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_PAIRS = """
|
||||
SELECT DISTINCT ON (shell.id) shell.id AS shell_id, real.id AS real_id
|
||||
FROM post shell
|
||||
JOIN source ss ON ss.id = shell.source_id
|
||||
JOIN post real
|
||||
ON real.artist_id = shell.artist_id
|
||||
AND real.external_post_id = shell.external_post_id
|
||||
AND real.id <> shell.id
|
||||
AND real.post_date IS NOT NULL
|
||||
JOIN source rs ON rs.id = real.source_id AND rs.platform = ss.platform
|
||||
WHERE shell.post_date IS NULL
|
||||
AND shell.post_url IS NULL
|
||||
AND shell.synthesized_by IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM image_provenance ip WHERE ip.post_id = shell.id)
|
||||
ORDER BY shell.id, real.id
|
||||
"""
|
||||
|
||||
|
||||
def fold_misfiled_attachment_posts(conn) -> int:
|
||||
"""The data step, on a plain connection, so a test can run it directly.
|
||||
Returns how many shells were folded."""
|
||||
pairs = conn.execute(sa.text(_PAIRS)).all()
|
||||
for shell_id, real_id in pairs:
|
||||
conn.execute(sa.text("""
|
||||
DELETE FROM post_attachment pa
|
||||
WHERE pa.post_id = :shell
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM post_attachment keep
|
||||
WHERE keep.post_id = :real AND keep.sha256 = pa.sha256
|
||||
)
|
||||
"""), {"shell": shell_id, "real": real_id})
|
||||
conn.execute(
|
||||
sa.text("UPDATE post_attachment SET post_id = :real WHERE post_id = :shell"),
|
||||
{"shell": shell_id, "real": real_id},
|
||||
)
|
||||
conn.execute(sa.text("DELETE FROM post WHERE id = :shell"), {"shell": shell_id})
|
||||
return len(pairs)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
fold_misfiled_attachment_posts(op.get_bind())
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
@@ -0,0 +1,27 @@
|
||||
"""source.display_name — what a source is called on its platform.
|
||||
|
||||
#4481. A Discord source's URL is a server id and a channel id, so the
|
||||
Subscriptions page could only show two numbers. The Discord ingester already
|
||||
loads the server and channel names to walk them; it now keeps them here,
|
||||
refreshed on every walk. NULL until a walk has read it.
|
||||
|
||||
Revision ID: 0115
|
||||
Revises: 0114
|
||||
Create Date: 2026-09-28
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0115"
|
||||
down_revision = "0114"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("source", sa.Column("display_name", sa.Text(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("source", "display_name")
|
||||
@@ -0,0 +1,34 @@
|
||||
"""The single-color filter's default becomes near-total: 0.95 → 0.995.
|
||||
|
||||
#4483. At 0.95 the filter rejected line art: a pencil doodle on white is
|
||||
mostly white, and five of Todding's Discord doodles were skipped on import as
|
||||
"single color", leaving their posts with text and no image. The predicate now
|
||||
samples real pixels instead of a blurred thumbnail, and the default only
|
||||
calls an image blank when it essentially is one.
|
||||
|
||||
A stored 0.95 is the old default, so it moves with it. Any other value was
|
||||
chosen by hand and is left alone.
|
||||
|
||||
Revision ID: 0116
|
||||
Revises: 0115
|
||||
Create Date: 2026-09-28
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
|
||||
revision = "0116"
|
||||
down_revision = "0115"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.alter_column("import_settings", "single_color_threshold", server_default="0.995")
|
||||
op.execute(
|
||||
"UPDATE import_settings SET single_color_threshold = 0.995 "
|
||||
"WHERE single_color_threshold = 0.95"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.alter_column("import_settings", "single_color_threshold", server_default="0.95")
|
||||
@@ -0,0 +1,33 @@
|
||||
"""external_link.started_at: when a fetch claimed the link.
|
||||
|
||||
A worker killed mid-fetch (the Celery hard limit, an OOM, a restart) never
|
||||
reaches the code that moves its link out of 'downloading'. The external-link
|
||||
sweep returns such a link to the retry path once it has been 'downloading'
|
||||
longer than the task's hard limit, which needs the claim time.
|
||||
|
||||
A row in 'downloading' with no claim time is treated as stale: no worker
|
||||
survives the deploy that adds this column.
|
||||
|
||||
Revision ID: 0117
|
||||
Revises: 0116
|
||||
Create Date: 2026-11-05
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0117"
|
||||
down_revision = "0116"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"external_link",
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("external_link", "started_at")
|
||||
@@ -0,0 +1,32 @@
|
||||
"""external_link.recoveries: how many times recovery has retried a dead link.
|
||||
|
||||
The daily recovery gives a dead link one more attempt, spaced further apart
|
||||
each time, and leaves it dead once the count reaches its limit. A link that
|
||||
can never succeed is then downloaded a handful of times in total, not every
|
||||
day.
|
||||
|
||||
Revision ID: 0118
|
||||
Revises: 0117
|
||||
Create Date: 2026-11-05
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0118"
|
||||
down_revision = "0117"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"external_link",
|
||||
sa.Column(
|
||||
"recoveries", sa.Integer(), nullable=False, server_default=sa.text("0")
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("external_link", "recoveries")
|
||||
@@ -0,0 +1,43 @@
|
||||
"""import_settings: the off-site backup destination (an S3 bucket).
|
||||
|
||||
Address, bucket, access key and region are plain settings; the secret key is
|
||||
stored sealed with the credential key. All empty means no destination is set
|
||||
up, which is every existing install's state.
|
||||
|
||||
Revision ID: 0119
|
||||
Revises: 0118
|
||||
Create Date: 2026-11-06
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0119"
|
||||
down_revision = "0118"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("import_settings", sa.Column("backup_s3_endpoint", sa.Text(), nullable=True))
|
||||
op.add_column(
|
||||
"import_settings", sa.Column("backup_s3_bucket", sa.String(255), nullable=True),
|
||||
)
|
||||
op.add_column("import_settings", sa.Column("backup_s3_access_key", sa.Text(), nullable=True))
|
||||
op.add_column(
|
||||
"import_settings", sa.Column("backup_s3_secret_sealed", sa.Text(), nullable=True),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"backup_s3_region", sa.String(64), nullable=False, server_default="us-east-1",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for column in (
|
||||
"backup_s3_region", "backup_s3_secret_sealed", "backup_s3_access_key",
|
||||
"backup_s3_bucket", "backup_s3_endpoint",
|
||||
):
|
||||
op.drop_column("import_settings", column)
|
||||
@@ -0,0 +1,30 @@
|
||||
"""import_settings: a bandwidth limit for uploads to the off-site backup bucket.
|
||||
|
||||
In megabits per second; 0 means no limit. Every install starts at 200 Mbit/s,
|
||||
which leaves room on a gigabit link for everything else the server does.
|
||||
|
||||
Revision ID: 0120
|
||||
Revises: 0119
|
||||
Create Date: 2026-11-07
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0120"
|
||||
down_revision = "0119"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"backup_upload_limit_mbps", sa.Integer(), nullable=False, server_default="200",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_settings", "backup_upload_limit_mbps")
|
||||
@@ -0,0 +1,46 @@
|
||||
"""backup_blob: which library files are already in the off-site bucket.
|
||||
|
||||
One row per (destination, sha256) sent. The media backup's work list is every
|
||||
image and attachment sha with no row for the current destination, so a run is
|
||||
incremental and resumable by construction, and pointing backups at a new
|
||||
bucket starts that bucket's copy from the beginning.
|
||||
|
||||
backup_run.last_progress_at is the heartbeat of a run that spans hours; the
|
||||
stall sweep measures a media run's silence from it, not from its start.
|
||||
|
||||
Revision ID: 0121
|
||||
Revises: 0120
|
||||
Create Date: 2026-11-07
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0121"
|
||||
down_revision = "0120"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"backup_blob",
|
||||
sa.Column("target", sa.Text(), nullable=False),
|
||||
sa.Column("sha256", sa.String(64), nullable=False),
|
||||
sa.Column("key", sa.Text(), nullable=False),
|
||||
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column(
|
||||
"uploaded_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.text("now()"),
|
||||
),
|
||||
sa.PrimaryKeyConstraint("target", "sha256", name=op.f("pk_backup_blob")),
|
||||
)
|
||||
op.add_column(
|
||||
"backup_run",
|
||||
sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("backup_run", "last_progress_at")
|
||||
op.drop_table("backup_blob")
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Retire the images tarball backup.
|
||||
|
||||
Images and attachments are backed up off-site, one object per file
|
||||
(backup_blob). The tarball of /images that this replaces is gone, with its
|
||||
retention setting, its path column, and any runs it left.
|
||||
|
||||
Revision ID: 0122
|
||||
Revises: 0121
|
||||
Create Date: 2026-11-08
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0122"
|
||||
down_revision = "0121"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("DELETE FROM backup_run WHERE kind = 'images'")
|
||||
op.drop_column("backup_run", "tar_path")
|
||||
op.drop_column("import_settings", "backup_images_keep_last_n")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.add_column("import_settings", sa.Column(
|
||||
"backup_images_keep_last_n", sa.Integer(), nullable=False, server_default="3"))
|
||||
op.add_column("backup_run", sa.Column("tar_path", sa.Text(), nullable=True))
|
||||
@@ -475,6 +475,21 @@ async def trigger_reclaim_attachments():
|
||||
return _queued(async_result)
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/import-unimported", methods=["POST"])
|
||||
async def trigger_import_unimported():
|
||||
"""Import files in the artists' folders that the library has no record of.
|
||||
Body {"dry_run": bool}; dry_run (the DEFAULT) lists what was found, per
|
||||
artist, and changes nothing. dry_run=false queues them as an import batch
|
||||
that attaches each file where it lies. Returns the Celery task id — poll
|
||||
/maintenance/task-result/<id> for the summary."""
|
||||
from ..tasks.admin import import_unimported_files_task
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", True))
|
||||
async_result = import_unimported_files_task.delay(dry_run=dry_run)
|
||||
return _queued(async_result)
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/repair-discord-downloads", methods=["POST"])
|
||||
async def trigger_repair_discord_downloads():
|
||||
"""Clean re-download of the Discord files broken by the `None` naming
|
||||
|
||||
@@ -19,8 +19,10 @@ from ..models import AppSetting
|
||||
from ..services.extension_service import (
|
||||
ExtensionService,
|
||||
InvalidUrlError,
|
||||
PosterLookupError,
|
||||
UnknownArtistError,
|
||||
UnknownPlatformError,
|
||||
UnknownSourceError,
|
||||
)
|
||||
from ..services.source_service import KNOWN_PLATFORMS
|
||||
from ._responses import error_response as _bad
|
||||
@@ -95,7 +97,9 @@ async def probe_source():
|
||||
return _bad("unauthorized", status=401)
|
||||
# crypto lets a Discord probe name the server and channel with the
|
||||
# stored token; every other platform ignores it.
|
||||
result = await ExtensionService(session, _get_crypto()).probe(url)
|
||||
result = await ExtensionService(session, _get_crypto()).probe(
|
||||
url, names=request.args.get("names") in ("1", "true"),
|
||||
)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@@ -116,6 +120,13 @@ async def quick_add_source():
|
||||
artist_name = body.get("artist_name")
|
||||
if artist_name is not None and not isinstance(artist_name, str):
|
||||
return _bad("invalid_body", detail="artist_name must be a string")
|
||||
# Patreon is canon: adding a Patreon source to an existing artist can take
|
||||
# the creator's Patreon display name (name only; the slug never moves).
|
||||
use_platform_name = body.get("use_platform_name") is True
|
||||
# #4491: a Discord add can carry the posters picked in the panel.
|
||||
discord_authors = body.get("discord_authors")
|
||||
if discord_authors is not None and not _valid_authors(discord_authors):
|
||||
return _bad("invalid_body", detail="discord_authors must be a list of {id, name}")
|
||||
|
||||
from .credentials import _get_crypto
|
||||
|
||||
@@ -127,6 +138,8 @@ async def quick_add_source():
|
||||
# stored credential (else it falls back to the URL handle). #130.
|
||||
result = await ExtensionService(session, _get_crypto()).quick_add_source(
|
||||
url, artist_id=artist_id, artist_name=artist_name,
|
||||
use_platform_name=use_platform_name,
|
||||
discord_authors=discord_authors,
|
||||
)
|
||||
except UnknownArtistError as exc:
|
||||
return _bad("not_found", detail=str(exc), status=404)
|
||||
@@ -141,6 +154,59 @@ async def quick_add_source():
|
||||
return jsonify(result), (201 if result["created_source"] else 200)
|
||||
|
||||
|
||||
def _valid_authors(value) -> bool:
|
||||
"""[{id, name}]: the id is what the list keys on, the name only shows it."""
|
||||
return isinstance(value, list) and all(
|
||||
isinstance(a, dict) and isinstance(a.get("id"), str | int)
|
||||
and not isinstance(a.get("id"), bool)
|
||||
and (a.get("name") is None or isinstance(a.get("name"), str))
|
||||
for a in value
|
||||
)
|
||||
|
||||
|
||||
@extension_bp.route("/discord/posters", methods=["GET"])
|
||||
async def discord_posters():
|
||||
"""Who posts in the Discord channel `url` shows, lately, with the likely
|
||||
creator marked (#4491). Reads the channel with the stored token; writes
|
||||
nothing."""
|
||||
url = (request.args.get("url") or "").strip()
|
||||
if not url:
|
||||
return _bad("invalid_body", detail="url query parameter is required")
|
||||
from .credentials import _get_crypto
|
||||
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_required(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
try:
|
||||
result = await ExtensionService(session, _get_crypto()).discord_posters(url)
|
||||
except (InvalidUrlError, UnknownPlatformError) as exc:
|
||||
return _bad("invalid_url", detail=str(exc))
|
||||
except PosterLookupError as exc:
|
||||
return _bad("posters_unavailable", detail=str(exc), status=409)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@extension_bp.route("/discord/posters", methods=["POST"])
|
||||
async def set_discord_posters():
|
||||
"""Set the poster list of the source following `url`'s channel."""
|
||||
body = await request.get_json(silent=True)
|
||||
if not isinstance(body, dict) or not isinstance(body.get("url"), str):
|
||||
return _bad("invalid_body", detail="url is required")
|
||||
authors = body.get("authors")
|
||||
if not _valid_authors(authors):
|
||||
return _bad("invalid_body", detail="authors must be a list of {id, name}")
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_required(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
try:
|
||||
result = await ExtensionService(session).set_discord_posters(body["url"], authors)
|
||||
except (InvalidUrlError, UnknownPlatformError, PosterLookupError) as exc:
|
||||
return _bad("invalid_url", detail=str(exc))
|
||||
except UnknownSourceError as exc:
|
||||
return _bad("not_found", detail=str(exc), status=404)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
def _read_manifest_sync() -> dict | None:
|
||||
"""All the filesystem-touching work for /api/extension/manifest,
|
||||
in a sync helper so the async route can dispatch it via
|
||||
|
||||
@@ -57,7 +57,6 @@ _EDITABLE_FIELDS = (
|
||||
"translation_target_lang",
|
||||
"translation_min_confidence",
|
||||
"wip_title_tagging_enabled",
|
||||
"wip_soft_title_tagging_enabled",
|
||||
)
|
||||
|
||||
# Per-host external-download toggles — all plain booleans, validated uniformly.
|
||||
@@ -196,12 +195,6 @@ async def update_import_settings():
|
||||
return jsonify(
|
||||
{"error": "wip_title_tagging_enabled must be a boolean"}
|
||||
), 400
|
||||
if "wip_soft_title_tagging_enabled" in body and not isinstance(
|
||||
body["wip_soft_title_tagging_enabled"], bool
|
||||
):
|
||||
return jsonify(
|
||||
{"error": "wip_soft_title_tagging_enabled must be a boolean"}
|
||||
), 400
|
||||
|
||||
async with get_session() as session:
|
||||
row = await ImportSettings.load(session)
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
"""FC-3a: CRUD over Source rows. FC-3c adds POST /<id>/check."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import DownloadEvent, MembershipSync, PlatformMembership, Source
|
||||
from ..services import discord_poster_cleanup as poster_cleanup
|
||||
from ..services.artist_membership_service import ArtistMembershipService
|
||||
from ..services.artist_membership_service import rescan as membership_rescan
|
||||
from ..services.artist_service import ArtistService
|
||||
@@ -463,3 +466,49 @@ async def adopt_membership():
|
||||
return jsonify({"already_tracked": exc.existing_id})
|
||||
artist_id = artist.id
|
||||
return jsonify({"source_id": record.id, "artist_id": artist_id}), 201
|
||||
|
||||
|
||||
# -- Discord: remove posts by people outside the poster list (#4486) --------------
|
||||
|
||||
_IMAGES_ROOT = Path("/images")
|
||||
|
||||
|
||||
async def _poster_cleanup_preview(source_id: int):
|
||||
async with get_session() as session:
|
||||
return await session.run_sync(
|
||||
lambda s: poster_cleanup.preview(s, source_id=source_id)
|
||||
)
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/discord/other-posters", methods=["GET"])
|
||||
async def other_posters_preview(source_id: int):
|
||||
"""What removing other posters' posts would delete. Nothing is touched."""
|
||||
try:
|
||||
projection = await _poster_cleanup_preview(source_id)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
except poster_cleanup.PosterCleanupError as exc:
|
||||
return _bad("not_applicable", detail=str(exc), status=409)
|
||||
projection["confirm_token"] = poster_cleanup.confirm_token(projection)
|
||||
return jsonify(projection)
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/discord/other-posters/remove", methods=["POST"])
|
||||
async def other_posters_remove(source_id: int):
|
||||
"""Delete them. `confirm` must be the token of the preview the operator saw:
|
||||
a poster list edited since then changes the set, and the token with it."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
try:
|
||||
projection = await _poster_cleanup_preview(source_id)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
except poster_cleanup.PosterCleanupError as exc:
|
||||
return _bad("not_applicable", detail=str(exc), status=409)
|
||||
expected = poster_cleanup.confirm_token(projection)
|
||||
if body.get("confirm") != expected:
|
||||
return _bad("confirm_mismatch", expected=expected)
|
||||
async with get_session() as session:
|
||||
result = await session.run_sync(
|
||||
lambda s: poster_cleanup.apply(s, source_id=source_id, images_root=_IMAGES_ROOT)
|
||||
)
|
||||
return jsonify(result)
|
||||
|
||||
@@ -152,6 +152,8 @@ async def list_runs():
|
||||
queue=<name> filter to one queue
|
||||
status=<status> filter to one status (running/ok/error/timeout/retry)
|
||||
task=<substr> case-insensitive substring match on task_name
|
||||
celery_task_id=<id> exactly one run — how a page follows a job it
|
||||
started without having to know its lane
|
||||
limit=<int> default 50, max 200
|
||||
before_id=<int> cursor for keyset pagination
|
||||
|
||||
@@ -167,6 +169,7 @@ async def list_runs():
|
||||
queue = request.args.get("queue")
|
||||
status = request.args.get("status")
|
||||
task = request.args.get("task")
|
||||
celery_task_id = request.args.get("celery_task_id")
|
||||
before_id_raw = request.args.get("before_id")
|
||||
before_id = int(before_id_raw) if before_id_raw else None
|
||||
|
||||
@@ -176,6 +179,8 @@ async def list_runs():
|
||||
stmt = stmt.where(TaskRun.queue == queue)
|
||||
if status:
|
||||
stmt = stmt.where(TaskRun.status == status)
|
||||
if celery_task_id:
|
||||
stmt = stmt.where(TaskRun.celery_task_id == celery_task_id)
|
||||
if task:
|
||||
# Task names contain literal underscores (download_source,
|
||||
# vacuum_analyze) — escape LIKE wildcards so a search for
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""FC-3h: /api/system/backup — create/list/restore/delete/tag for
|
||||
DB + image backups.
|
||||
"""/api/system/backup: database dumps, the off-site destination and its media
|
||||
backup, restores, test restores, and the run history.
|
||||
|
||||
Read endpoints are public on FC (operator-facing internal API; same
|
||||
posture as /api/system/activity). Write endpoints take a typed
|
||||
@@ -20,13 +20,12 @@ system_backup_bp = Blueprint(
|
||||
"system_backup", __name__, url_prefix="/api/system/backup",
|
||||
)
|
||||
|
||||
_KINDS = frozenset({"db", "images"})
|
||||
_KINDS = frozenset({"db", "media", "restore", "reconcile", "test"})
|
||||
_TAG_MAX_LEN = 64
|
||||
_BACKUP_SETTINGS_FIELDS = (
|
||||
"backup_db_nightly_enabled",
|
||||
"backup_db_nightly_hour_utc",
|
||||
"backup_db_keep_last_n",
|
||||
"backup_images_keep_last_n",
|
||||
)
|
||||
|
||||
|
||||
@@ -44,11 +43,13 @@ def _row_to_dict(r: BackupRun) -> dict:
|
||||
if r.finished_at and r.started_at else None
|
||||
),
|
||||
"sql_path": r.sql_path,
|
||||
"tar_path": r.tar_path,
|
||||
"size_bytes": r.size_bytes,
|
||||
"error": r.error,
|
||||
"restored_from_id": r.restored_from_id,
|
||||
"manifest": r.manifest or {},
|
||||
# Where an off-site dump went ({endpoint, bucket, key}); None for a
|
||||
# backup kept on this server.
|
||||
"offsite": (r.manifest or {}).get("offsite"),
|
||||
}
|
||||
|
||||
|
||||
@@ -78,10 +79,6 @@ def _validate_backup_settings_patch(body: dict):
|
||||
v = body["backup_db_keep_last_n"]
|
||||
if not isinstance(v, int) or isinstance(v, bool) or not (1 <= v <= 365):
|
||||
return _bad("invalid_value", detail="backup_db_keep_last_n must be 1..365")
|
||||
if "backup_images_keep_last_n" in body:
|
||||
v = body["backup_images_keep_last_n"]
|
||||
if not isinstance(v, int) or isinstance(v, bool) or not (1 <= v <= 100):
|
||||
return _bad("invalid_value", detail="backup_images_keep_last_n must be 1..100")
|
||||
return None
|
||||
|
||||
|
||||
@@ -96,14 +93,173 @@ async def trigger_db_backup():
|
||||
return jsonify({"status": "dispatched"}), 202
|
||||
|
||||
|
||||
@system_backup_bp.route("/images", methods=["POST"])
|
||||
async def trigger_images_backup():
|
||||
@system_backup_bp.route("/media", methods=["POST"])
|
||||
async def trigger_media_backup():
|
||||
"""Send the images and attachments not yet in the off-site bucket. Refused
|
||||
when no destination is set up; a run already going is left to finish."""
|
||||
from ..services import backup_destination as dest
|
||||
|
||||
async with get_session() as session:
|
||||
row = await ImportSettings.load(session)
|
||||
if not dest.describe(row)["configured"]:
|
||||
return _bad("no_destination",
|
||||
detail="set up an off-site destination before sending files")
|
||||
from ..tasks.backup import backup_media_task
|
||||
backup_media_task.delay(triggered_by="manual")
|
||||
return jsonify({"status": "dispatched"}), 202
|
||||
|
||||
|
||||
@system_backup_bp.route("/media", methods=["GET"])
|
||||
async def media_coverage():
|
||||
"""How much of the library is in the off-site bucket: files and bytes,
|
||||
sent and total, each file counted once however many posts hold it."""
|
||||
from ..services import backup_destination as dest
|
||||
from ..services import backup_media
|
||||
|
||||
async with get_session() as session:
|
||||
row = await ImportSettings.load(session)
|
||||
f = dest.fields_of(row)
|
||||
configured = dest.describe(row)["configured"]
|
||||
tid = backup_media.destination_id(f.endpoint, f.bucket) if configured else None
|
||||
counts = await session.run_sync(lambda s: backup_media.coverage(s, tid))
|
||||
return jsonify({"configured": configured, **counts})
|
||||
|
||||
|
||||
# ── Restoring from the off-site bucket ──────────────────────────────────────
|
||||
|
||||
|
||||
async def _saved_target():
|
||||
"""(target, None) for the saved destination, or (None, error response)."""
|
||||
from ..services import backup_destination as dest
|
||||
from ..services.backup_s3 import S3Error
|
||||
|
||||
async with get_session() as session:
|
||||
row = await ImportSettings.load(session)
|
||||
try:
|
||||
target = dest.saved_target(row)
|
||||
except S3Error as exc:
|
||||
return None, _bad("destination_unreadable", detail=str(exc))
|
||||
if target is None:
|
||||
return None, _bad("no_destination",
|
||||
detail="set up an off-site destination before restoring from it")
|
||||
return target, None
|
||||
|
||||
|
||||
async def _busy_reason() -> str | None:
|
||||
from ..tasks.backup import backup_busy
|
||||
|
||||
async with get_session() as session:
|
||||
return await session.run_sync(backup_busy)
|
||||
|
||||
|
||||
def restore_token(ts: str) -> str:
|
||||
"""What the person types to confirm replacing the database with the dump
|
||||
taken at `ts`."""
|
||||
return f"restore-offsite-{ts}"
|
||||
|
||||
|
||||
@system_backup_bp.route("/offsite/dumps", methods=["GET"])
|
||||
async def list_offsite_dumps():
|
||||
"""The database dumps in the off-site bucket, newest first, each with
|
||||
the confirmation a restore of it asks for."""
|
||||
import asyncio
|
||||
|
||||
from ..services import backup_restore
|
||||
from ..services.backup_s3 import S3Error
|
||||
|
||||
target, err = await _saved_target()
|
||||
if err:
|
||||
return err
|
||||
try:
|
||||
dumps = await asyncio.to_thread(backup_restore.list_dumps, target)
|
||||
except S3Error as exc:
|
||||
return _bad("destination_unreachable", detail=str(exc), status=502)
|
||||
return jsonify({
|
||||
"dumps": [
|
||||
{"key": d.key, "ts": d.ts, "size_bytes": d.size_bytes, "counted": d.counted,
|
||||
"confirm": restore_token(d.ts)}
|
||||
for d in dumps
|
||||
],
|
||||
# A test restore needs about this much free space on the database's
|
||||
# disk while it runs.
|
||||
"database_bytes": await _database_bytes(),
|
||||
})
|
||||
|
||||
|
||||
async def _database_bytes() -> int:
|
||||
from sqlalchemy import text
|
||||
|
||||
async with get_session() as session:
|
||||
return (await session.execute(
|
||||
text("SELECT pg_database_size(current_database())"))).scalar_one()
|
||||
|
||||
|
||||
async def _restore_test_blocked() -> str | None:
|
||||
from ..tasks.backup import restore_test_blocked
|
||||
|
||||
async with get_session() as session:
|
||||
return await session.run_sync(restore_test_blocked)
|
||||
|
||||
|
||||
@system_backup_bp.route("/offsite/test", methods=["POST"])
|
||||
async def restore_test_offsite():
|
||||
"""Test a dump from the bucket without touching this install: it is
|
||||
loaded into a temporary database, checked table by table, and dropped.
|
||||
Body: {key}, a dump's key from /offsite/dumps, or {} for the newest.
|
||||
Needs no confirmation; refused while a restore or another test is going."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
tag = _validate_tag(body.get("tag"))
|
||||
if isinstance(tag, tuple):
|
||||
return tag
|
||||
from ..tasks.backup import backup_images_task
|
||||
backup_images_task.delay(tag=tag, triggered_by="manual")
|
||||
key = body.get("key")
|
||||
if key is not None and (not isinstance(key, str) or not key.startswith("database/")):
|
||||
return _bad("invalid_key", detail="key must be a dump's key from /offsite/dumps")
|
||||
_, err = await _saved_target()
|
||||
if err:
|
||||
return err
|
||||
busy = await _restore_test_blocked()
|
||||
if busy:
|
||||
return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
|
||||
from ..tasks.backup import restore_test_task
|
||||
restore_test_task.delay(dump_key=key)
|
||||
return jsonify({"status": "dispatched"}), 202
|
||||
|
||||
|
||||
@system_backup_bp.route("/offsite/restore", methods=["POST"])
|
||||
async def restore_from_offsite():
|
||||
"""Replace this install's database with a dump from the bucket, then
|
||||
fetch the library's files back. Body: {key, confirm}, where confirm is
|
||||
the dump's token from /offsite/dumps. Refused while a backup or another
|
||||
restore is going."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
key = body.get("key")
|
||||
if not isinstance(key, str) or not key.startswith("database/"):
|
||||
return _bad("invalid_key", detail="key must be a dump's key from /offsite/dumps")
|
||||
ts = key.removeprefix("database/fabledcurator-").removesuffix(".dump")
|
||||
expected = restore_token(ts)
|
||||
if body.get("confirm") != expected:
|
||||
return _bad("confirm_mismatch", detail=f"confirm must equal {expected!r}",
|
||||
expected=expected)
|
||||
_, err = await _saved_target()
|
||||
if err:
|
||||
return err
|
||||
busy = await _busy_reason()
|
||||
if busy:
|
||||
return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
|
||||
from ..tasks.backup import restore_offsite_task
|
||||
restore_offsite_task.delay(dump_key=key)
|
||||
return jsonify({"status": "dispatched"}), 202
|
||||
|
||||
|
||||
@system_backup_bp.route("/offsite/fetch-files", methods=["POST"])
|
||||
async def fetch_missing_files():
|
||||
"""Fetch every library file that is missing from disk back from the
|
||||
bucket, leaving the database as it is."""
|
||||
_, err = await _saved_target()
|
||||
if err:
|
||||
return err
|
||||
busy = await _busy_reason()
|
||||
if busy:
|
||||
return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
|
||||
from ..tasks.backup import restore_offsite_task
|
||||
restore_offsite_task.delay(files_only=True)
|
||||
return jsonify({"status": "dispatched"}), 202
|
||||
|
||||
|
||||
@@ -175,10 +331,16 @@ async def trigger_restore(run_id: int):
|
||||
row = await session.get(BackupRun, run_id)
|
||||
if row is None:
|
||||
return _bad("not_found", status=404)
|
||||
if row.status != "ok":
|
||||
if (row.manifest or {}).get("offsite"):
|
||||
return _bad(
|
||||
"not_restorable",
|
||||
detail=f"source backup status={row.status!r}; only 'ok' rows are restorable",
|
||||
detail=("this dump is kept in the off-site bucket, not on this server, "
|
||||
"so it can't be restored from here"),
|
||||
)
|
||||
if row.kind != "db" or row.status != "ok" or not row.sql_path:
|
||||
return _bad(
|
||||
"not_restorable",
|
||||
detail="only a finished database dump kept on this server can be restored here",
|
||||
)
|
||||
expected = f"restore-{row.kind}-{row.id}"
|
||||
if supplied != expected:
|
||||
@@ -187,15 +349,29 @@ async def trigger_restore(run_id: int):
|
||||
detail=f"confirm must equal {expected!r}",
|
||||
expected=expected,
|
||||
)
|
||||
kind = row.kind
|
||||
|
||||
if kind == "db":
|
||||
from ..tasks.backup import restore_db_task
|
||||
restore_db_task.delay(source_backup_run_id=run_id)
|
||||
else: # 'images' (the only other value _KINDS allows via the trigger path)
|
||||
from ..tasks.backup import restore_images_task
|
||||
restore_images_task.delay(source_backup_run_id=run_id)
|
||||
return jsonify({"status": "dispatched", "kind": kind}), 202
|
||||
from ..tasks.backup import restore_db_task
|
||||
restore_db_task.delay(source_backup_run_id=run_id)
|
||||
return jsonify({"status": "dispatched", "kind": "db"}), 202
|
||||
|
||||
|
||||
@system_backup_bp.route("/runs/<int:run_id>/test", methods=["POST"])
|
||||
async def restore_test_local(run_id: int):
|
||||
"""Test a database dump kept on this server, the same way as one in the
|
||||
bucket. Needs no confirmation."""
|
||||
async with get_session() as session:
|
||||
row = await session.get(BackupRun, run_id)
|
||||
if row is None:
|
||||
return _bad("not_found", status=404)
|
||||
if row.kind != "db" or row.status != "ok" or not row.sql_path:
|
||||
return _bad("not_testable",
|
||||
detail="only a finished database dump on this server can be tested here")
|
||||
busy = await _restore_test_blocked()
|
||||
if busy:
|
||||
return _bad("busy", detail=f"{busy}; try again when it has finished", status=409)
|
||||
from ..tasks.backup import restore_test_task
|
||||
restore_test_task.delay(source_backup_run_id=run_id)
|
||||
return jsonify({"status": "dispatched"}), 202
|
||||
|
||||
|
||||
@system_backup_bp.route("/runs/<int:run_id>", methods=["DELETE"])
|
||||
@@ -216,7 +392,7 @@ async def delete_run(run_id: int):
|
||||
)
|
||||
from ..services import backup_service
|
||||
backup_service.unlink_artifact_files(
|
||||
sql_path=row.sql_path, tar_path=row.tar_path,
|
||||
sql_path=row.sql_path,
|
||||
manifest_path=(row.manifest or {}).get("manifest_path"),
|
||||
)
|
||||
await session.delete(row)
|
||||
@@ -232,10 +408,121 @@ async def get_settings():
|
||||
"backup_db_nightly_enabled": row.backup_db_nightly_enabled,
|
||||
"backup_db_nightly_hour_utc": row.backup_db_nightly_hour_utc,
|
||||
"backup_db_keep_last_n": row.backup_db_keep_last_n,
|
||||
"backup_images_keep_last_n": row.backup_images_keep_last_n,
|
||||
})
|
||||
|
||||
|
||||
@system_backup_bp.route("/destination", methods=["GET"])
|
||||
async def get_destination():
|
||||
"""The off-site destination as the settings screen shows it. The secret
|
||||
key is never returned, only whether one is saved."""
|
||||
from ..services import backup_destination as dest
|
||||
|
||||
async with get_session() as session:
|
||||
row = await ImportSettings.load(session)
|
||||
return jsonify(dest.describe(row))
|
||||
|
||||
|
||||
@system_backup_bp.route("/destination", methods=["PATCH"])
|
||||
async def patch_destination():
|
||||
"""Save the destination. Body {endpoint, bucket, access_key, region,
|
||||
secret_key, upload_limit_mbps}; an empty field clears it, and a missing
|
||||
upload_limit_mbps keeps the saved limit. A missing or empty secret_key
|
||||
keeps the saved one, so the screen can save the other fields without the
|
||||
secret being typed again."""
|
||||
from ..services import backup_destination as dest
|
||||
|
||||
body = await request.get_json(silent=True)
|
||||
if not isinstance(body, dict):
|
||||
return _bad("invalid_body", detail="body must be a JSON object")
|
||||
fields, why = dest.clean_fields(body)
|
||||
if fields is None:
|
||||
return _bad("invalid_destination", detail=why)
|
||||
secret = body.get("secret_key")
|
||||
if secret is not None and not isinstance(secret, str):
|
||||
return _bad("invalid_destination", detail="secret_key must be text")
|
||||
limit = body.get("upload_limit_mbps")
|
||||
if limit is not None and (
|
||||
not isinstance(limit, int) or isinstance(limit, bool) or not 0 <= limit <= 100_000
|
||||
):
|
||||
return _bad("invalid_destination",
|
||||
detail="upload_limit_mbps must be a whole number from 0 (no limit) up")
|
||||
|
||||
async with get_session() as session:
|
||||
row = await ImportSettings.load(session)
|
||||
if limit is not None:
|
||||
row.backup_upload_limit_mbps = limit
|
||||
row.backup_s3_endpoint = fields.endpoint
|
||||
row.backup_s3_bucket = fields.bucket
|
||||
row.backup_s3_access_key = fields.access_key
|
||||
row.backup_s3_region = fields.region
|
||||
if secret and secret.strip():
|
||||
row.backup_s3_secret_sealed = dest.seal(secret.strip())
|
||||
await session.commit()
|
||||
return jsonify(dest.describe(row))
|
||||
|
||||
|
||||
@system_backup_bp.route("/destination", methods=["DELETE"])
|
||||
async def delete_destination():
|
||||
"""Forget the destination, secret included. What was already sent stays in
|
||||
the bucket: FabledCurator never deletes there."""
|
||||
from ..services import backup_destination as dest
|
||||
|
||||
async with get_session() as session:
|
||||
row = await ImportSettings.load(session)
|
||||
row.backup_s3_endpoint = None
|
||||
row.backup_s3_bucket = None
|
||||
row.backup_s3_access_key = None
|
||||
row.backup_s3_secret_sealed = None
|
||||
row.backup_s3_region = dest.DEFAULT_REGION
|
||||
await session.commit()
|
||||
return jsonify(dest.describe(row))
|
||||
|
||||
|
||||
@system_backup_bp.route("/destination/test", methods=["POST"])
|
||||
async def try_destination():
|
||||
"""Reach the bucket with the fields in the body (the form as typed, saved
|
||||
or not); a missing secret_key uses the saved one. Lists the bucket and
|
||||
writes nothing. Answers {ok, message} either way: a destination that
|
||||
can't be reached is a result to show, not an API error."""
|
||||
import asyncio
|
||||
|
||||
from ..services import backup_destination as dest
|
||||
from ..services.backup_s3 import S3Error
|
||||
from ..services.credential_crypto import InvalidCredentialBlob
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
if not isinstance(body, dict):
|
||||
return _bad("invalid_body", detail="body must be a JSON object")
|
||||
fields, why = dest.clean_fields(body)
|
||||
if fields is None:
|
||||
return jsonify({"ok": False, "message": f"Not tested: {why}."})
|
||||
secret = body.get("secret_key")
|
||||
secret = secret.strip() if isinstance(secret, str) else ""
|
||||
if not secret:
|
||||
async with get_session() as session:
|
||||
row = await ImportSettings.load(session)
|
||||
sealed = row.backup_s3_secret_sealed
|
||||
if sealed:
|
||||
try:
|
||||
secret = dest.unseal(sealed)
|
||||
except InvalidCredentialBlob:
|
||||
return jsonify({"ok": False, "message": (
|
||||
"Not tested: the saved secret key can't be opened on this server. "
|
||||
"Enter the secret key again."
|
||||
)})
|
||||
target = dest.target_for(fields, secret or None)
|
||||
if target is None:
|
||||
missing = [name for name, v in (
|
||||
("address", fields.endpoint), ("bucket", fields.bucket),
|
||||
("access key", fields.access_key), ("secret key", secret),
|
||||
) if not v]
|
||||
return jsonify({"ok": False, "message": f"Not tested: fill in the {', '.join(missing)}."})
|
||||
try:
|
||||
return jsonify(await asyncio.to_thread(dest.check_target, target))
|
||||
except S3Error as exc:
|
||||
return jsonify({"ok": False, "message": str(exc)})
|
||||
|
||||
|
||||
@system_backup_bp.route("/settings", methods=["PATCH"])
|
||||
async def patch_settings():
|
||||
body = await request.get_json(silent=True)
|
||||
|
||||
@@ -35,6 +35,7 @@ from sqlalchemy import select, text
|
||||
from ..config import get_config
|
||||
from ..extensions import get_session
|
||||
from ..models import ServiceSeen
|
||||
from ..services import backup_health
|
||||
from ..services.worker_lanes import SWEEP_PERIOD_SECONDS
|
||||
|
||||
system_health_bp = Blueprint("system_health", __name__, url_prefix="/api/system")
|
||||
@@ -227,6 +228,17 @@ async def system_health():
|
||||
**{k: v for k, v in (row.details or {}).items() if k != "agent_id"},
|
||||
})
|
||||
|
||||
# Whether the backups are keeping up. Read like the rest, and a
|
||||
# failure to read it is reported as unknown rather than raised.
|
||||
try:
|
||||
backup = await session.run_sync(lambda s: backup_health.backup_part(s, now))
|
||||
except Exception as exc: # noqa: BLE001 — a probe reports, it never raises
|
||||
backup = {"key": "backup", "kind": "backup", "name": "Backups",
|
||||
"state": _UNKNOWN,
|
||||
"detail": f"could not be checked: {type(exc).__name__}"}
|
||||
if backup is not None:
|
||||
parts.append(backup)
|
||||
|
||||
parts.append(await _probe_redis())
|
||||
|
||||
overall = max((p["state"] for p in parts), key=lambda s: _SEVERITY[s], default=_UNKNOWN)
|
||||
|
||||
@@ -4,7 +4,7 @@ Queues:
|
||||
import — filesystem import path (FC-2)
|
||||
ml — WD14 + SigLIP inference (FC-2; runs in the ml-worker image)
|
||||
thumbnail — image and video thumbnail generation (FC-2)
|
||||
download — gallery-dl tasks (FC-3)
|
||||
download — source walks: the native ingesters and gallery-dl (FC-3)
|
||||
scan — periodic source checks (FC-3) — kept separate so long imports
|
||||
don't starve the scheduler
|
||||
maintenance — recovery sweeps, pHash backfill, GPU-queue coordination, etc.
|
||||
@@ -69,6 +69,8 @@ def make_celery() -> Celery:
|
||||
# up behind it (2026-09-24: 7 waiting, "all workers busy for 18
|
||||
# minutes"). An exact name wins over the glob above.
|
||||
"backend.app.tasks.maintenance.backfill_phash": {"queue": "maintenance_long"},
|
||||
# Walks a folder tree per artist with undated posts (#4436).
|
||||
"backend.app.tasks.maintenance.date_posts_from_records": {"queue": "maintenance_long"},
|
||||
"backend.app.tasks.backup.*": {"queue": "maintenance_long"},
|
||||
"backend.app.tasks.admin.*": {"queue": "maintenance_long"},
|
||||
"backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
|
||||
@@ -156,6 +158,11 @@ def make_celery() -> Celery:
|
||||
"schedule": 86400.0, # daily — sweep .part/.partial left by a
|
||||
# download/import killed mid-write (graceful-shutdown fallout)
|
||||
},
|
||||
"date-posts-from-records-hourly": {
|
||||
"task": "backend.app.tasks.maintenance.date_posts_from_records",
|
||||
"schedule": 3600.0, # an empty query once every native post is
|
||||
# dated; otherwise upserts the _post.json a killed walk left (#4436)
|
||||
},
|
||||
"backfill-phash-daily": {
|
||||
"task": "backend.app.tasks.maintenance.backfill_phash",
|
||||
"schedule": 86400.0, # daily — NULL-only, so a no-op once the
|
||||
@@ -223,11 +230,6 @@ def make_celery() -> Celery:
|
||||
"schedule": 86400.0, # auto-tag wip/editor process art (#1464);
|
||||
# no-op unless process_auto_apply_enabled (opt-in)
|
||||
},
|
||||
"soft-wip-conflict-audit-daily": {
|
||||
"task": "backend.app.tasks.ml.scheduled_soft_wip_conflict_audit",
|
||||
"schedule": 86400.0, # flag ring-loud soft-WIP (sketch/doodle) tags
|
||||
# for review (#1474); no-op with no content heads
|
||||
},
|
||||
"prune-presentation-reviews-daily": {
|
||||
"task": "backend.app.tasks.ml.prune_presentation_reviews",
|
||||
"schedule": 86400.0, # retention: drop resolved review flags >30d
|
||||
@@ -300,6 +302,10 @@ def make_celery() -> Celery:
|
||||
"task": "backend.app.tasks.backup.prune_backups",
|
||||
"schedule": 86400.0, # daily
|
||||
},
|
||||
"reconcile-media-backup": {
|
||||
"task": "backend.app.tasks.backup.reconcile_media_backup",
|
||||
"schedule": 604800.0, # weekly: list the bucket, re-send what's missing
|
||||
},
|
||||
# Audit 2026-06-02 — three new per-entity recovery sweeps.
|
||||
# Each runs every 5 min like the other recover_stalled_*
|
||||
# sweeps; each is a no-op when nothing is stuck.
|
||||
|
||||
@@ -18,11 +18,18 @@ dark for that interval. Monitoring NEVER breaks the thing it's
|
||||
monitoring.
|
||||
"""
|
||||
|
||||
import functools
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
from celery.signals import task_failure, task_postrun, task_prerun, task_retry
|
||||
from celery.signals import (
|
||||
task_failure,
|
||||
task_postrun,
|
||||
task_prerun,
|
||||
task_retry,
|
||||
worker_ready,
|
||||
)
|
||||
|
||||
from .models import TaskRun
|
||||
from .tasks._sync_engine import sync_session_factory
|
||||
@@ -53,42 +60,29 @@ _INT32_MIN = -2_147_483_648
|
||||
|
||||
|
||||
def _queue_for(task) -> str:
|
||||
"""Reverse the task→queue routing from celery_app.task_routes.
|
||||
Keep in sync if task_routes is reordered.
|
||||
"""The queue Celery routes this task to — asked of the router itself.
|
||||
|
||||
Audit 2026-06-02: backup/admin/library_audit prefixes were
|
||||
missing here even though task_routes sent all three to
|
||||
'maintenance'. The TaskRun.queue column then lied for those
|
||||
rows (claimed 'default') so per-queue dashboard filters and
|
||||
per-queue threshold overrides silently missed them.
|
||||
This was a hand-kept copy of `celery_app.task_routes`, and it drifted
|
||||
twice (the 2026-06-02 audit, then #4432). Long-lane jobs were recorded as
|
||||
`maintenance`, and translation and gpu_queue runs as `default`, where the
|
||||
5-minute stall sweep failed healthy 35-minute translation runs. The router
|
||||
answers from the same table the broker uses, so the two cannot disagree.
|
||||
"""
|
||||
name = getattr(task, "name", "") or ""
|
||||
if name.startswith("backend.app.tasks.import_file."):
|
||||
return "import"
|
||||
if name.startswith("backend.app.tasks.ml."):
|
||||
return "ml"
|
||||
if name.startswith("backend.app.tasks.thumbnail."):
|
||||
return "thumbnail"
|
||||
if name.startswith((
|
||||
"backend.app.tasks.download.",
|
||||
# External file-host fetches share the download lane (celery_app
|
||||
# routes external.* → download). Mirror it here or TaskRun.queue
|
||||
# lies 'default' for them, so per-queue dashboard filters and the
|
||||
# per-queue threshold override miss them — the same gap the
|
||||
# 2026-06-02 audit fixed for backup/admin/library_audit.
|
||||
"backend.app.tasks.external.",
|
||||
)):
|
||||
return "download"
|
||||
if name.startswith("backend.app.tasks.scan."):
|
||||
return "scan"
|
||||
if name.startswith((
|
||||
"backend.app.tasks.maintenance.",
|
||||
"backend.app.tasks.backup.",
|
||||
"backend.app.tasks.admin.",
|
||||
"backend.app.tasks.library_audit.",
|
||||
)):
|
||||
return "maintenance"
|
||||
return "default"
|
||||
app = getattr(task, "app", None)
|
||||
if app is None:
|
||||
from .celery_app import celery as app
|
||||
return _routed_queue(app, name)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1024)
|
||||
def _routed_queue(app, name: str) -> str:
|
||||
try:
|
||||
queue = app.amqp.router.route({}, name).get("queue")
|
||||
except Exception: # noqa: BLE001 — monitoring never breaks the task
|
||||
log.warning("task_run: could not resolve the queue for %s", name)
|
||||
return "default"
|
||||
return getattr(queue, "name", None) or (queue if isinstance(queue, str) else "default")
|
||||
|
||||
|
||||
def _target_id_from_args(args) -> int | None:
|
||||
@@ -225,3 +219,64 @@ def _on_retry(sender=None, request=None, reason=None, einfo=None, **_):
|
||||
error_message=str(reason) if reason else None,
|
||||
retry_count=getattr(request, "retries", 0),
|
||||
)
|
||||
|
||||
|
||||
def _consumed_queues(consumer) -> set[str]:
|
||||
"""The queue names this worker process consumes (its `-Q`), or empty when
|
||||
they can't be read — which makes the boot hook below a no-op, not a guess."""
|
||||
try:
|
||||
return {q.name for q in consumer.task_consumer.queues}
|
||||
except Exception: # noqa: BLE001 — shape varies across celery versions
|
||||
return set()
|
||||
|
||||
|
||||
@worker_ready.connect
|
||||
def _on_worker_ready(sender=None, **_):
|
||||
"""A lane picks up what its previous process left behind. Only the lane
|
||||
that runs the work does it; the other lanes booting beside it must not,
|
||||
or a lane restarting on its own would disturb work still running.
|
||||
Best-effort: a failure is logged and the worker still starts."""
|
||||
queues = _consumed_queues(sender)
|
||||
if "download" in queues:
|
||||
_clear_download_leftovers()
|
||||
if "maintenance_long" in queues:
|
||||
_resume_offsite_runs()
|
||||
|
||||
|
||||
def _resume_offsite_runs() -> None:
|
||||
"""The long-maintenance lane carries on an off-site backup or restore that
|
||||
a restart cut off mid-chunk. With nothing queuing its next chunk, the run
|
||||
would read "running" for hours, and block every new run, until the stall
|
||||
sweep ended it."""
|
||||
try:
|
||||
from .tasks.backup import resume_offsite_runs
|
||||
|
||||
resume_offsite_runs()
|
||||
except Exception: # noqa: BLE001 — never block the worker from starting
|
||||
log.exception("long-maintenance lane boot: resuming off-site runs failed")
|
||||
|
||||
|
||||
def _clear_download_leftovers() -> None:
|
||||
"""The download lane clears what its previous process left behind.
|
||||
|
||||
A restart SIGKILLs any walk that outlives the stop grace, so its event is
|
||||
never finalized and its platform lock is never released. Both used to wait
|
||||
out timers — a 30-min stall sweep that then blamed the source, and a 27-min
|
||||
lock TTL that stalled every other source on the platform.
|
||||
"""
|
||||
booted_at = datetime.now(UTC)
|
||||
try:
|
||||
from .services.download_recovery import interrupt_orphaned_download_events
|
||||
from .services.platform_lock import release_all_platform_locks
|
||||
|
||||
with sync_session_factory()() as session:
|
||||
closed = interrupt_orphaned_download_events(session, booted_at=booted_at)
|
||||
session.commit()
|
||||
released = release_all_platform_locks()
|
||||
if closed or released:
|
||||
log.info(
|
||||
"download lane boot: closed %d orphaned download event(s) as "
|
||||
"interrupted, released %d platform lock(s)", closed, released,
|
||||
)
|
||||
except Exception: # noqa: BLE001 — never block the worker from starting
|
||||
log.exception("download lane boot recovery failed")
|
||||
|
||||
+20
-2
@@ -46,6 +46,14 @@ async def serve_extension(filename: str):
|
||||
|
||||
The application/x-xpinstall MIME tells Firefox to show its native
|
||||
install prompt instead of downloading the file as a blob.
|
||||
|
||||
Caching differs by name, and has to. A versioned name is one build's bytes
|
||||
forever, so it can be cached for good. `fabledcurator-latest.xpi` is ONE
|
||||
URL whose bytes change on every release, and Quart's default for a file
|
||||
is `public, max-age=43200`: a browser that fetched it once reused those
|
||||
bytes for 12 hours, so "install the latest" quietly reinstalled the
|
||||
previous build (operator-flagged 2026-09-25). It is `no-cache` — the ETag
|
||||
still makes an unchanged file a cheap 304.
|
||||
"""
|
||||
if not _XPI_NAME_RE.fullmatch(filename):
|
||||
abort(404)
|
||||
@@ -56,10 +64,11 @@ async def serve_extension(filename: str):
|
||||
if not xpis:
|
||||
abort(404)
|
||||
latest = xpis[-1]
|
||||
return await send_file(
|
||||
resp = await send_file(
|
||||
latest, mimetype="application/x-xpinstall",
|
||||
attachment_filename=latest.name,
|
||||
)
|
||||
return _cache(resp, "no-cache")
|
||||
target = (XPI_DIR / filename).resolve()
|
||||
try:
|
||||
target.relative_to(XPI_DIR)
|
||||
@@ -67,10 +76,19 @@ async def serve_extension(filename: str):
|
||||
abort(404)
|
||||
if not target.is_file():
|
||||
abort(404)
|
||||
return await send_file(
|
||||
resp = await send_file(
|
||||
target, mimetype="application/x-xpinstall",
|
||||
attachment_filename=filename,
|
||||
)
|
||||
return _cache(resp, "public, max-age=31536000, immutable")
|
||||
|
||||
|
||||
def _cache(resp, policy: str):
|
||||
"""Set the XPI's Cache-Control, dropping the Expires send_file adds so the
|
||||
two can never disagree."""
|
||||
resp.headers["Cache-Control"] = policy
|
||||
resp.headers.pop("Expires", None)
|
||||
return resp
|
||||
|
||||
|
||||
@frontend_bp.route("/")
|
||||
|
||||
@@ -4,6 +4,7 @@ from .app_setting import AppSetting
|
||||
from .artist import Artist
|
||||
from .artist_membership_suggestion import ArtistMembershipSuggestion
|
||||
from .artist_visit import ArtistVisit
|
||||
from .backup_blob import BackupBlob
|
||||
from .backup_run import BackupRun
|
||||
from .base import Base
|
||||
from .character_prototype import CcipPrototypeState, CharacterPrototype
|
||||
@@ -55,6 +56,7 @@ __all__ = [
|
||||
"Artist",
|
||||
"ArtistMembershipSuggestion",
|
||||
"ArtistVisit",
|
||||
"BackupBlob",
|
||||
"BackupRun",
|
||||
"Source",
|
||||
"Credential",
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
"""backup_blob: a library file already sent to the off-site bucket.
|
||||
|
||||
One row per (destination, sha256). `target` names the destination as
|
||||
"<endpoint>/<bucket>", so pointing backups at another bucket leaves these rows
|
||||
inert and that bucket's copy starts from the beginning. `key` is where the
|
||||
file sits in the bucket (`media/ab/cd/<sha256><ext>`).
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import BigInteger, DateTime, String, Text, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class BackupBlob(Base):
|
||||
__tablename__ = "backup_blob"
|
||||
|
||||
target: Mapped[str] = mapped_column(Text, primary_key=True)
|
||||
sha256: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
key: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
uploaded_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now(),
|
||||
)
|
||||
@@ -1,6 +1,8 @@
|
||||
"""FC-3h: backup_run — operator-facing artifact record for a backup run.
|
||||
|
||||
One row per backup attempt (kind='db' or 'images'). Lifecycle
|
||||
One row per run: kind 'db' (a database dump), 'media' (the off-site upload of
|
||||
images and attachments, which spans many task chunks), 'restore', 'reconcile'
|
||||
(the weekly bucket check) or 'test' (a test restore). Lifecycle
|
||||
tracking (started_at/finished_at/duration_ms/exception text) lives
|
||||
in task_run from FC-3i — this row records artifact metadata: file
|
||||
paths, sizes, tag (retention protection), and restore lineage via
|
||||
@@ -54,7 +56,6 @@ class BackupRun(Base):
|
||||
DateTime(timezone=True), nullable=True, index=True,
|
||||
)
|
||||
sql_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
tar_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
size_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
manifest: Mapped[dict] = mapped_column(
|
||||
@@ -62,6 +63,11 @@ class BackupRun(Base):
|
||||
)
|
||||
# Self-referential FK, unindexed until 0089 (#3300): SET NULL has to find
|
||||
# the rows pointing at a deleted run before it can null them.
|
||||
# Heartbeat of a run that spans hours (the media backup): set as it makes
|
||||
# progress, and what the stall sweep measures its silence from.
|
||||
last_progress_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True,
|
||||
)
|
||||
restored_from_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("backup_run.id", ondelete="SET NULL"),
|
||||
nullable=True, index=True,
|
||||
|
||||
@@ -80,12 +80,23 @@ class ExternalLink(Base):
|
||||
Integer, nullable=False, server_default=text("0")
|
||||
)
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
# Times the daily recovery has given this link another attempt after it
|
||||
# went dead. Recovery stops at a limit, so a link that can never succeed
|
||||
# isn't downloaded forever.
|
||||
recoveries: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, server_default=text("0")
|
||||
)
|
||||
attachment_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("post_attachment.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
# When the current fetch claimed the link ('downloading'). The sweep uses it
|
||||
# to recover a link whose worker died mid-fetch.
|
||||
started_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
|
||||
@@ -9,6 +9,7 @@ from sqlalchemy import (
|
||||
CheckConstraint,
|
||||
Float,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
select,
|
||||
text,
|
||||
@@ -39,7 +40,7 @@ class ImportSettings(Base):
|
||||
transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9, server_default="0.9")
|
||||
|
||||
skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false")
|
||||
single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95, server_default="0.95")
|
||||
single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.995, server_default="0.995")
|
||||
single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30, server_default="30")
|
||||
|
||||
# Hamming distance over a 256-bit pHash (utils.phash, hash_size=16). The
|
||||
@@ -96,9 +97,19 @@ class ImportSettings(Base):
|
||||
Integer, nullable=False, default=14,
|
||||
server_default="14",
|
||||
)
|
||||
backup_images_keep_last_n: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=3,
|
||||
server_default="3",
|
||||
# Off-site backup destination: an S3 bucket (services/backup_destination).
|
||||
# The secret key is stored sealed with the credential key, never in clear.
|
||||
backup_s3_endpoint: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
backup_s3_bucket: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
backup_s3_access_key: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
backup_s3_secret_sealed: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
backup_s3_region: Mapped[str] = mapped_column(
|
||||
String(64), nullable=False, default="us-east-1",
|
||||
server_default="us-east-1",
|
||||
)
|
||||
# Upload bandwidth to that bucket, in megabits per second; 0 = no limit.
|
||||
backup_upload_limit_mbps: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=200, server_default="200",
|
||||
)
|
||||
|
||||
# FC-6.3 series continuation matcher. enabled gates the rescan; threshold is
|
||||
@@ -238,13 +249,6 @@ class ImportSettings(Base):
|
||||
wip_title_tagging_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True, server_default="true",
|
||||
)
|
||||
# Soft WIP title tier (#1474): also tag sketch/doodle/scribble titles, but with
|
||||
# a PROVISIONAL source (`wip_title_soft`) that never trains the head, since these
|
||||
# are lower-precision (a finished "sketch" isn't WIP). OFF by default — a lower-
|
||||
# precision tier is opt-in (the ring-loud audit surfaces false positives).
|
||||
wip_soft_title_tagging_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="false",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def load(cls, session) -> ImportSettings:
|
||||
|
||||
@@ -46,6 +46,11 @@ class Source(Base):
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true")
|
||||
|
||||
config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
# alembic 0115: what the source is called on its platform, where the URL
|
||||
# alone doesn't say — a Discord link is two numbers. The ingester that
|
||||
# walks it refreshes it every walk, so a rename reads as the new name.
|
||||
# NULL until a walk has read it, and on platforms that don't write it.
|
||||
display_name: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Restore FabledCurator from its off-site bucket, from a shell in the
|
||||
container. The Backup screen does the same; this is for an install whose page
|
||||
you can't reach yet, or for scripting a rebuild.
|
||||
|
||||
python -m backend.app.scripts.restore list
|
||||
python -m backend.app.scripts.restore database --dump latest --yes
|
||||
python -m backend.app.scripts.restore files
|
||||
|
||||
On a brand-new install with no destination saved, name it on the command
|
||||
line: --endpoint, --bucket and --access-key, with the secret key in the
|
||||
FC_BACKUP_SECRET_KEY environment variable (or typed at the prompt). It is
|
||||
checked by listing the bucket, then saved as this install's destination,
|
||||
sealed, exactly as the settings page saves it.
|
||||
|
||||
`database` replaces the database with the chosen dump here, then hands the
|
||||
files to the worker; their progress shows on the Backup screen. `files`
|
||||
fetches only what is missing from disk and leaves the database alone.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import getpass
|
||||
import os
|
||||
import sys
|
||||
|
||||
from ..models import ImportSettings
|
||||
from ..services import backup_destination, backup_restore
|
||||
from ..services.backup_s3 import S3Error
|
||||
from ..tasks._sync_engine import sync_session_factory
|
||||
|
||||
|
||||
def _human(n: int) -> str:
|
||||
for unit in ("B", "KB", "MB", "GB", "TB"):
|
||||
if n < 1000 or unit == "TB":
|
||||
return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}"
|
||||
n /= 1000
|
||||
return str(n)
|
||||
|
||||
|
||||
def _destination(args) -> object:
|
||||
"""The target to restore from: the one named on the command line, saved
|
||||
first, or the one already saved."""
|
||||
session_factory = sync_session_factory()
|
||||
if args.endpoint or args.bucket or args.access_key:
|
||||
fields, why = backup_destination.clean_fields({
|
||||
"endpoint": args.endpoint, "bucket": args.bucket,
|
||||
"access_key": args.access_key, "region": args.region,
|
||||
})
|
||||
if fields is None:
|
||||
raise SystemExit(f"Destination not usable: {why}.")
|
||||
secret = os.environ.get("FC_BACKUP_SECRET_KEY") or getpass.getpass("Secret key: ")
|
||||
target = backup_destination.target_for(fields, secret.strip() or None)
|
||||
if target is None:
|
||||
raise SystemExit("Name the address, bucket, access key and secret key.")
|
||||
check = backup_destination.check_target(target)
|
||||
print(check["message"])
|
||||
if not check["ok"]:
|
||||
raise SystemExit(1)
|
||||
with session_factory() as session:
|
||||
settings = ImportSettings.load_sync(session)
|
||||
limit = settings.backup_upload_limit_mbps
|
||||
backup_restore.keep_destination(session, target, limit)
|
||||
print("Saved as this install's off-site destination.")
|
||||
return target
|
||||
with session_factory() as session:
|
||||
target = backup_destination.saved_target(ImportSettings.load_sync(session))
|
||||
if target is None:
|
||||
raise SystemExit("No off-site destination is saved; name one with "
|
||||
"--endpoint, --bucket and --access-key.")
|
||||
return target
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(prog="python -m backend.app.scripts.restore",
|
||||
description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("action", choices=("list", "database", "files"))
|
||||
parser.add_argument("--dump", default="latest",
|
||||
help="the dump's key from `list`, or latest (the default)")
|
||||
parser.add_argument("--yes", action="store_true",
|
||||
help="confirm replacing this install's database")
|
||||
parser.add_argument("--endpoint")
|
||||
parser.add_argument("--bucket")
|
||||
parser.add_argument("--access-key")
|
||||
parser.add_argument("--region", default=None)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
target = _destination(args)
|
||||
if args.action == "list":
|
||||
for d in backup_restore.list_dumps(target):
|
||||
print(f"{d.key} {_human(d.size_bytes)}")
|
||||
return 0
|
||||
|
||||
from ..tasks.backup import restore_offsite_task
|
||||
if args.action == "files":
|
||||
restore_offsite_task.delay(files_only=True)
|
||||
print("Queued: the worker fetches every missing file; watch the Backup screen.")
|
||||
return 0
|
||||
|
||||
dump = backup_restore.find_dump(target, args.dump)
|
||||
if not args.yes:
|
||||
print(f"This replaces this install's database with {dump.key} "
|
||||
f"({_human(dump.size_bytes)}). Run again with --yes to go ahead.")
|
||||
return 2
|
||||
print(f"Restoring {dump.key}; this takes a while for a large library…")
|
||||
result = restore_offsite_task(dump_key=dump.key)
|
||||
if result.get("skipped"):
|
||||
print(f"Not started: {result['skipped']}.")
|
||||
return 1
|
||||
print("The database is restored. The worker is fetching the library's files; "
|
||||
"watch the Backup screen.")
|
||||
return 0
|
||||
except S3Error as exc:
|
||||
print(f"Couldn't restore: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -57,6 +57,50 @@ def is_archive(path: Path) -> bool:
|
||||
return detect_archive_format(path) is not None
|
||||
|
||||
|
||||
# Files that only a program ships: executables and libraries, and the packed
|
||||
# data and compiled scripts of the common game engines (Ren'Py, Unity, Godot,
|
||||
# RPG Maker, Python builds). An archive holding any of them is a game or app
|
||||
# build. Its images are sprites, UI and CGs bundled for the program, not art
|
||||
# for the library, so the archive is kept whole as an attachment.
|
||||
SOFTWARE_EXTS = frozenset({
|
||||
".exe", ".msi", ".dll", ".so", ".dylib", ".apk",
|
||||
".rpa", ".rpyc", ".rpy", ".pyc", ".pyo",
|
||||
".pak", ".pck", ".unity3d", ".assets", ".ress",
|
||||
".rgssad", ".rgss2a", ".rgss3a", ".rxdata", ".rvdata", ".rvdata2",
|
||||
})
|
||||
|
||||
|
||||
def member_names(path: Path) -> list[str]:
|
||||
"""Every member name in the archive, read from its directory without
|
||||
decompressing anything. Empty when the archive can't be listed."""
|
||||
try:
|
||||
fmt = detect_archive_format(path)
|
||||
if fmt == "zip":
|
||||
with zipfile.ZipFile(path) as zf:
|
||||
return zf.namelist()
|
||||
if fmt == "rar":
|
||||
import rarfile
|
||||
|
||||
with rarfile.RarFile(path) as rf:
|
||||
return rf.namelist()
|
||||
if fmt == "7z":
|
||||
import py7zr
|
||||
|
||||
with py7zr.SevenZipFile(path, "r") as zf:
|
||||
return zf.getnames()
|
||||
except Exception as exc:
|
||||
log.warning("archive listing failed for %s: %s", path, exc)
|
||||
return []
|
||||
|
||||
|
||||
def software_marker(path: Path) -> str | None:
|
||||
"""The first member that marks the archive as a software build, or None."""
|
||||
for name in member_names(path):
|
||||
if Path(name).suffix.lower() in SOFTWARE_EXTS:
|
||||
return name
|
||||
return None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def extract_archive(path: Path):
|
||||
"""Context manager yielding a list of (member_name, extracted_path)
|
||||
|
||||
@@ -316,17 +316,32 @@ class ArtistService:
|
||||
cleaned = (prefix or "").strip()
|
||||
if not cleaned:
|
||||
return []
|
||||
like = f"%{cleaned.lower()}%"
|
||||
prefix_like = f"{cleaned.lower()}%"
|
||||
# Rank: exact (0) < prefix (1) < substring (2).
|
||||
low = cleaned.lower()
|
||||
like = f"%{low}%"
|
||||
prefix_like = f"{low}%"
|
||||
# Spacing- and punctuation-insensitive too, so "Tamada Heijun" finds
|
||||
# "TamadaHeijun" and "sabu_art" finds "Sabu Art" — the same creator is
|
||||
# spelled differently on every platform, and the browser extension's
|
||||
# Add panel matches a Discord server name against these (milestone 429).
|
||||
# [[:alnum:]] keeps non-Latin letters; a query with none (all
|
||||
# punctuation) skips this arm rather than matching every artist.
|
||||
squashed = "".join(ch for ch in low if ch.isalnum())
|
||||
name_squashed = func.regexp_replace(func.lower(Artist.name), "[^[:alnum:]]", "", "g")
|
||||
matches = [func.lower(Artist.name).like(like)]
|
||||
if squashed:
|
||||
matches.append(name_squashed.like(f"%{squashed}%"))
|
||||
# Rank: exact (0) < exact ignoring spacing (1) < prefix (2) <
|
||||
# substring (3) < substring ignoring spacing (4).
|
||||
rank = case(
|
||||
(func.lower(Artist.name) == cleaned.lower(), 0),
|
||||
(func.lower(Artist.name).like(prefix_like), 1),
|
||||
else_=2,
|
||||
(func.lower(Artist.name) == low, 0),
|
||||
(name_squashed == squashed, 1),
|
||||
(func.lower(Artist.name).like(prefix_like), 2),
|
||||
(func.lower(Artist.name).like(like), 3),
|
||||
else_=4,
|
||||
).label("rank")
|
||||
rows = (await self.session.execute(
|
||||
select(Artist, rank)
|
||||
.where(func.lower(Artist.name).like(like))
|
||||
.where(or_(*matches))
|
||||
.order_by(rank, Artist.name.asc())
|
||||
.limit(limit)
|
||||
)).all()
|
||||
|
||||
@@ -4,11 +4,19 @@ predicate for BOTH surfaces: FC-Cleanup's retroactive audit and — since
|
||||
2026-07-02 — the import-side filter (Importer._single_color_hit /
|
||||
SkipReason.single_color), so what the audit flags and what the import
|
||||
skips can never disagree.
|
||||
|
||||
It is meant to catch the blank: a placeholder, an error tile, a solid fill.
|
||||
Line art is the case it must not catch (#4483): a pencil doodle on white can be
|
||||
well over 95% white even at full size, and a smoothing downsample blends its strokes
|
||||
into the paper until it measures as blank. So the sample is taken by NEAREST
|
||||
(each sampled pixel is a real pixel, strokes keep their contrast), at 256px
|
||||
so thin strokes are still hit, and the default threshold is near-total
|
||||
(0.995) — a few percent of ink is a drawing, not an empty image.
|
||||
"""
|
||||
|
||||
from PIL import Image
|
||||
|
||||
_THUMB_SIZE = (64, 64)
|
||||
_THUMB_SIZE = (256, 256)
|
||||
|
||||
|
||||
def evaluate(
|
||||
@@ -20,7 +28,8 @@ def evaluate(
|
||||
"""True iff the fraction of pixels within `tolerance` (Euclidean RGB
|
||||
distance) of the dominant color exceeds `threshold`.
|
||||
|
||||
Downsamples to 64x64 for speed (~4ms regardless of source size).
|
||||
Samples 256x256 by NEAREST (see the module docstring for why not a
|
||||
smoothing resample).
|
||||
Alpha channels are stripped; only RGB is considered. Animated images
|
||||
use frame 0 (PIL's default after Image.open without seek).
|
||||
"""
|
||||
@@ -30,7 +39,7 @@ def evaluate(
|
||||
elif im.mode not in ("RGB", "L"):
|
||||
im = im.convert("RGB")
|
||||
if im.size != _THUMB_SIZE:
|
||||
im = im.resize(_THUMB_SIZE, Image.Resampling.BILINEAR)
|
||||
im = im.resize(_THUMB_SIZE, Image.Resampling.NEAREST)
|
||||
pixels = list(im.getdata())
|
||||
if not pixels:
|
||||
return False
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
"""The off-site backup destination: an S3 bucket (Fabled Archives, or any S3
|
||||
service) that the database dumps and the media blobs are sent to.
|
||||
|
||||
The address, bucket, access key and region are plain settings. The secret key
|
||||
is sealed at rest with the credential key (`/images/secrets`), the same key
|
||||
that seals account logins, and no API ever returns it: the settings answer
|
||||
only whether one is set. Because the credential key is kept out of every
|
||||
backup, a database dump carries the secret only as ciphertext nobody can open
|
||||
without the original server.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from .backup_s3 import S3Error, S3Target, list_objects
|
||||
from .credential_crypto import CredentialCrypto, InvalidCredentialBlob
|
||||
|
||||
KEY_PATH = Path("/images/secrets/credential_key.b64")
|
||||
DEFAULT_REGION = "us-east-1"
|
||||
_TEST_LISTING = 5
|
||||
|
||||
|
||||
def _crypto() -> CredentialCrypto:
|
||||
return CredentialCrypto(KEY_PATH)
|
||||
|
||||
|
||||
def seal(secret: str) -> str:
|
||||
return _crypto().encrypt(secret).decode("ascii")
|
||||
|
||||
|
||||
def unseal(sealed: str) -> str:
|
||||
return _crypto().decrypt(sealed.encode("ascii"))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DestinationFields:
|
||||
endpoint: str | None
|
||||
bucket: str | None
|
||||
access_key: str | None
|
||||
region: str
|
||||
|
||||
|
||||
def clean_fields(body: dict) -> tuple[DestinationFields | None, str | None]:
|
||||
"""The destination fields from a request body, trimmed, with empty ones as
|
||||
None; or (None, why) when the address can't be an S3 endpoint."""
|
||||
def text(name: str) -> str | None:
|
||||
v = body.get(name)
|
||||
if v is None:
|
||||
return None
|
||||
if not isinstance(v, str):
|
||||
raise TypeError(name)
|
||||
return v.strip() or None
|
||||
|
||||
try:
|
||||
endpoint = text("endpoint")
|
||||
bucket = text("bucket")
|
||||
access_key = text("access_key")
|
||||
region = text("region") or DEFAULT_REGION
|
||||
except TypeError as exc:
|
||||
return None, f"{exc.args[0]} must be text"
|
||||
if endpoint is not None:
|
||||
u = urlsplit(endpoint)
|
||||
if u.scheme not in ("http", "https") or not u.hostname:
|
||||
return None, ("the address must start with http:// or https:// and name "
|
||||
"a host, e.g. http://archives.example:8081")
|
||||
if u.query or u.fragment:
|
||||
return None, "the address can't carry a query or a fragment"
|
||||
endpoint = endpoint.rstrip("/")
|
||||
if bucket is not None and "/" in bucket:
|
||||
return None, "the bucket is a name, not a path: put any path in the address"
|
||||
return DestinationFields(endpoint, bucket, access_key, region), None
|
||||
|
||||
|
||||
def target_for(fields: DestinationFields, secret: str | None) -> S3Target | None:
|
||||
"""The S3 target, or None while any part of it is missing."""
|
||||
if not (fields.endpoint and fields.bucket and fields.access_key and secret):
|
||||
return None
|
||||
return S3Target(fields.endpoint, fields.bucket, fields.access_key, secret,
|
||||
region=fields.region)
|
||||
|
||||
|
||||
def saved_target(settings) -> S3Target | None:
|
||||
"""The destination saved in ImportSettings, or None when none is set up.
|
||||
Raises S3Error when the secret can't be opened (the credential key
|
||||
changed since it was saved)."""
|
||||
sealed = settings.backup_s3_secret_sealed
|
||||
if not sealed:
|
||||
return None
|
||||
try:
|
||||
secret = unseal(sealed)
|
||||
except InvalidCredentialBlob:
|
||||
raise S3Error(
|
||||
"The saved secret key can't be opened on this server (the credential "
|
||||
"key changed since it was saved). Enter the secret key again."
|
||||
) from None
|
||||
return target_for(fields_of(settings), secret)
|
||||
|
||||
|
||||
def fields_of(settings) -> DestinationFields:
|
||||
return DestinationFields(
|
||||
settings.backup_s3_endpoint, settings.backup_s3_bucket,
|
||||
settings.backup_s3_access_key, settings.backup_s3_region or DEFAULT_REGION,
|
||||
)
|
||||
|
||||
|
||||
def describe(settings) -> dict:
|
||||
"""The destination as the settings screen shows it: never the secret."""
|
||||
f = fields_of(settings)
|
||||
return {
|
||||
"endpoint": f.endpoint or "",
|
||||
"bucket": f.bucket or "",
|
||||
"access_key": f.access_key or "",
|
||||
"region": f.region,
|
||||
"secret_set": bool(settings.backup_s3_secret_sealed),
|
||||
"upload_limit_mbps": settings.backup_upload_limit_mbps,
|
||||
"configured": bool(
|
||||
f.endpoint and f.bucket and f.access_key and settings.backup_s3_secret_sealed
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def check_target(target: S3Target) -> dict:
|
||||
"""Reach the bucket by listing it. A listing proves the address, the
|
||||
bucket and both keys at once, and writes nothing, because an Archives key
|
||||
is allowed to add objects but a test object could never be removed."""
|
||||
try:
|
||||
page = list_objects(target, limit=_TEST_LISTING)
|
||||
except S3Error as exc:
|
||||
return {"ok": False, "message": str(exc)}
|
||||
if not page.objects:
|
||||
said = f"Reached {target.bucket} at {target.endpoint}. The bucket is empty."
|
||||
else:
|
||||
more = "+" if page.next_token else ""
|
||||
said = (f"Reached {target.bucket} at {target.endpoint}. It holds "
|
||||
f"{len(page.objects)}{more} object(s), e.g. {page.objects[0].key}.")
|
||||
return {"ok": True, "message": said}
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Are the backups keeping up? One row on the System view, and the nav dot.
|
||||
|
||||
The backups run on their own at night, so the only way to notice one has
|
||||
stopped is to be told. The System view's health endpoint asks `backup_part`,
|
||||
which answers with a part like any other: `ok`, or `stale` with the sentence
|
||||
that says what is wrong. It warns when
|
||||
|
||||
- the newest database backup is more than a day old, or there is none;
|
||||
- the newest database backup failed;
|
||||
- the weekly check of the off-site bucket found files missing from it, and no
|
||||
media run has finished since to send them again.
|
||||
|
||||
No part at all when backups aren't set up (no off-site destination and the
|
||||
nightly backup off): someone who chose not to back up is not told so every
|
||||
fifteen seconds.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..models import BackupRun, ImportSettings
|
||||
from . import backup_destination
|
||||
|
||||
# A nightly backup is due every 24 hours. The hour of slack covers the dump
|
||||
# itself and a beat tick that lands a little late.
|
||||
DUMP_DUE_SECONDS = 25 * 3600
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackupFacts:
|
||||
expected: bool
|
||||
offsite: bool
|
||||
last_dump_at: datetime | None = None
|
||||
last_dump_error: str | None = None
|
||||
missing: int = 0
|
||||
|
||||
|
||||
def _age(seconds: float) -> str:
|
||||
hours = int(seconds // 3600)
|
||||
if hours >= 48:
|
||||
return f"{hours // 24} days"
|
||||
if hours >= 1:
|
||||
return f"{hours} h"
|
||||
return f"{max(1, int(seconds // 60))} min"
|
||||
|
||||
|
||||
def verdict(facts: BackupFacts, now: datetime) -> dict | None:
|
||||
"""The part for these facts, or None when backups aren't set up."""
|
||||
if not facts.expected:
|
||||
return None
|
||||
where = "to the off-site bucket" if facts.offsite else "on this server"
|
||||
problems = []
|
||||
if facts.last_dump_error:
|
||||
problems.append(f"The last database backup failed: {facts.last_dump_error}")
|
||||
if facts.last_dump_at is None:
|
||||
if not facts.last_dump_error:
|
||||
problems.append(f"No database backup has been taken {where} yet")
|
||||
else:
|
||||
age = (now - facts.last_dump_at).total_seconds()
|
||||
if age >= DUMP_DUE_SECONDS:
|
||||
problems.append(f"The newest database backup {where} is {_age(age)} old")
|
||||
if facts.missing:
|
||||
problems.append(
|
||||
f"The bucket check found {facts.missing:,} file(s) missing from the bucket; "
|
||||
"the next media run sends them again"
|
||||
)
|
||||
part = {"key": "backup", "kind": "backup", "name": "Backups"}
|
||||
if problems:
|
||||
return {**part, "state": "stale", "detail": ". ".join(problems)}
|
||||
age = (now - facts.last_dump_at).total_seconds()
|
||||
return {**part, "state": "ok",
|
||||
"detail": f"The newest database backup {where} is {_age(age)} old"}
|
||||
|
||||
|
||||
def gather(session) -> BackupFacts:
|
||||
"""Read the facts `verdict` judges, from a sync session."""
|
||||
settings = ImportSettings.load_sync(session)
|
||||
offsite = backup_destination.describe(settings)["configured"]
|
||||
expected = offsite or bool(settings.backup_db_nightly_enabled)
|
||||
if not expected:
|
||||
return BackupFacts(expected=False, offsite=False)
|
||||
|
||||
def newest(kind: str, statuses: tuple[str, ...]):
|
||||
return session.execute(
|
||||
select(BackupRun).where(BackupRun.kind == kind)
|
||||
.where(BackupRun.status.in_(statuses))
|
||||
.order_by(BackupRun.started_at.desc()).limit(1)
|
||||
).scalar_one_or_none()
|
||||
|
||||
last_ok = newest("db", ("ok",))
|
||||
last_done = newest("db", ("ok", "error"))
|
||||
failed = None
|
||||
if last_done is not None and last_done.status == "error":
|
||||
failed = last_done.error or "no reason recorded"
|
||||
|
||||
missing = 0
|
||||
if offsite:
|
||||
check = newest("reconcile", ("ok",))
|
||||
dropped = int((check.manifest or {}).get("dropped") or 0) if check else 0
|
||||
if dropped:
|
||||
sent_since = session.execute(
|
||||
select(BackupRun.id).where(BackupRun.kind == "media")
|
||||
.where(BackupRun.status == "ok")
|
||||
.where(BackupRun.finished_at > check.finished_at).limit(1)
|
||||
).scalar_one_or_none()
|
||||
missing = 0 if sent_since is not None else dropped
|
||||
|
||||
return BackupFacts(
|
||||
expected=True, offsite=offsite,
|
||||
last_dump_at=(last_ok.finished_at or last_ok.started_at) if last_ok else None,
|
||||
last_dump_error=failed, missing=missing,
|
||||
)
|
||||
|
||||
|
||||
def backup_part(session, now: datetime) -> dict | None:
|
||||
return verdict(gather(session), now)
|
||||
@@ -0,0 +1,202 @@
|
||||
"""The off-site copy of the library's files: every image and attachment, sent
|
||||
once each to the backup bucket as a content-addressed object.
|
||||
|
||||
Layout in the bucket, beside the database dumps under `database/`:
|
||||
|
||||
media/ab/cd/<sha256><ext>
|
||||
|
||||
`ab` and `cd` are the first four hex digits of the file's sha256, which keeps
|
||||
any one prefix small, and the extension keeps the files readable in the bucket
|
||||
without FabledCurator. The sha256 is the one recorded at import, so a file
|
||||
uploads exactly once however many rows or posts point at it, and a run is
|
||||
incremental: its work list is every sha with no `backup_blob` row for the
|
||||
current destination. A file removed from the library stays in the bucket until
|
||||
the bucket's own retention removes it; FabledCurator never deletes there.
|
||||
|
||||
The bucket checks every object against that recorded sha256, so a file whose
|
||||
bytes changed on disk since import is refused and reported, never stored under
|
||||
the wrong name.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import delete, select, text
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
|
||||
from ..models import BackupBlob
|
||||
from .backup_s3 import (
|
||||
Pacer,
|
||||
S3Error,
|
||||
S3Target,
|
||||
list_objects,
|
||||
object_present,
|
||||
put_file,
|
||||
put_file_in_parts,
|
||||
)
|
||||
|
||||
PREFIX = "media/"
|
||||
# Files above this size go in parts (S3 refuses a single PUT over 5 GiB, and a
|
||||
# part that fails is resent alone instead of the whole file).
|
||||
MULTIPART_OVER = 1 << 30
|
||||
# The bucket is listed this many keys at a time during a reconcile, with a
|
||||
# short rest between pages.
|
||||
LIST_PAGE = 1000
|
||||
LIST_REST_SECONDS = 0.2
|
||||
_EXT = re.compile(r"^\.[a-z0-9]{1,10}$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Blob:
|
||||
sha256: str
|
||||
path: str
|
||||
size_bytes: int
|
||||
mime: str | None
|
||||
|
||||
|
||||
def destination_id(endpoint: str, bucket: str) -> str:
|
||||
"""How backup_blob names a destination: one copy per endpoint and bucket."""
|
||||
return f"{endpoint.rstrip('/')}/{bucket}"
|
||||
|
||||
|
||||
def target_id(target: S3Target) -> str:
|
||||
return destination_id(target.endpoint, target.bucket)
|
||||
|
||||
|
||||
def blob_key(sha256: str, path: str) -> str:
|
||||
"""The bucket key for a file: `media/ab/cd/<sha256><ext>`. An extension
|
||||
that isn't short and plain is left off, so a key is always safe."""
|
||||
ext = Path(path).suffix.lower()
|
||||
return f"{PREFIX}{sha256[:2]}/{sha256[2:4]}/{sha256}{ext if _EXT.match(ext) else ''}"
|
||||
|
||||
|
||||
# Every library file once per sha256: images and attachments together. An
|
||||
# attachment's sha can repeat across posts, and an image extracted from an
|
||||
# archive is its own file with its own sha.
|
||||
_LIBRARY = """
|
||||
SELECT sha256, path, size_bytes, mime FROM image_record
|
||||
UNION ALL
|
||||
SELECT sha256, path, size_bytes, mime FROM post_attachment
|
||||
"""
|
||||
|
||||
|
||||
def pending(session, tid: str, *, after: str = "", limit: int = 500) -> list[Blob]:
|
||||
"""The next `limit` files not yet in the bucket, in sha order after
|
||||
`after`, so a run walks the library once even when some files fail."""
|
||||
rows = session.execute(text(f"""
|
||||
SELECT DISTINCT ON (lib.sha256) lib.sha256, lib.path, lib.size_bytes, lib.mime
|
||||
FROM ({_LIBRARY}) AS lib
|
||||
WHERE lib.sha256 > :after
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM backup_blob b WHERE b.target = :tid AND b.sha256 = lib.sha256
|
||||
)
|
||||
ORDER BY lib.sha256, lib.path
|
||||
LIMIT :limit
|
||||
"""), {"after": after, "tid": tid, "limit": limit}).all()
|
||||
return [Blob(r.sha256, r.path, r.size_bytes, r.mime) for r in rows]
|
||||
|
||||
|
||||
def coverage(session, tid: str | None) -> dict:
|
||||
"""How much of the library is in the bucket: files and bytes, sent and
|
||||
total, each sha counted once."""
|
||||
row = session.execute(text(f"""
|
||||
WITH lib AS (
|
||||
SELECT DISTINCT ON (sha256) sha256, size_bytes FROM ({_LIBRARY}) AS u
|
||||
ORDER BY sha256
|
||||
)
|
||||
SELECT count(*) AS files_total,
|
||||
coalesce(sum(lib.size_bytes), 0) AS bytes_total,
|
||||
count(b.sha256) AS files_sent,
|
||||
coalesce(sum(lib.size_bytes) FILTER (WHERE b.sha256 IS NOT NULL), 0)
|
||||
AS bytes_sent
|
||||
FROM lib
|
||||
LEFT JOIN backup_blob b ON b.sha256 = lib.sha256 AND b.target = :tid
|
||||
"""), {"tid": tid or ""}).one()
|
||||
return {
|
||||
"files_total": row.files_total, "bytes_total": int(row.bytes_total),
|
||||
"files_sent": row.files_sent, "bytes_sent": int(row.bytes_sent),
|
||||
}
|
||||
|
||||
|
||||
def send(target: S3Target, blob: Blob, images_root: Path, pacer: Pacer | None) -> str:
|
||||
"""Upload one file to its key and return the key. Raises S3Error (or
|
||||
OSError) with the reason when it can't be sent."""
|
||||
path = Path(blob.path)
|
||||
if not path.is_absolute():
|
||||
path = images_root / path
|
||||
if not path.is_file():
|
||||
raise S3Error(f"{path} is not on disk.")
|
||||
size = path.stat().st_size
|
||||
if size != blob.size_bytes:
|
||||
raise S3Error(f"{path} is {size} bytes on disk but {blob.size_bytes} when imported.")
|
||||
key = blob_key(blob.sha256, blob.path)
|
||||
content_type = blob.mime or "application/octet-stream"
|
||||
try:
|
||||
if size > MULTIPART_OVER:
|
||||
put_file_in_parts(target, key, path, sha256=blob.sha256,
|
||||
content_type=content_type, pacer=pacer)
|
||||
else:
|
||||
put_file(target, key, path, content_type=content_type,
|
||||
payload_sha256=blob.sha256, pacer=pacer)
|
||||
except S3Error:
|
||||
if object_present(target, key, size):
|
||||
return key # already there from an earlier run
|
||||
raise
|
||||
return key
|
||||
|
||||
|
||||
def record(session, tid: str, blobs: list[tuple[Blob, str]]) -> None:
|
||||
"""Note files as sent: (blob, key) pairs. A row already there is kept."""
|
||||
if blobs:
|
||||
session.execute(insert(BackupBlob).values([
|
||||
{"target": tid, "sha256": b.sha256, "key": key, "size_bytes": b.size_bytes}
|
||||
for b, key in blobs
|
||||
]).on_conflict_do_nothing())
|
||||
|
||||
|
||||
def reconcile(session, target: S3Target) -> dict:
|
||||
"""Check backup_blob against what the bucket actually holds under
|
||||
`media/`. A row whose object is missing or the wrong size is dropped, so
|
||||
the next run sends that file again; an object the bucket holds for a
|
||||
library file with no row (a restored database, a lost commit) is recorded,
|
||||
so it isn't sent twice. Returns the counts."""
|
||||
tid = target_id(target)
|
||||
held: dict[str, int] = {}
|
||||
token = None
|
||||
while True:
|
||||
page = list_objects(target, PREFIX, LIST_PAGE, continuation=token)
|
||||
held.update((o.key, o.size) for o in page.objects)
|
||||
token = page.next_token
|
||||
if not token:
|
||||
break
|
||||
time.sleep(LIST_REST_SECONDS)
|
||||
|
||||
rows = session.execute(
|
||||
select(BackupBlob.sha256, BackupBlob.key, BackupBlob.size_bytes)
|
||||
.where(BackupBlob.target == tid)
|
||||
).all()
|
||||
gone = [r.sha256 for r in rows if held.get(r.key) != r.size_bytes]
|
||||
for at in range(0, len(gone), 1000):
|
||||
session.execute(delete(BackupBlob).where(
|
||||
BackupBlob.target == tid, BackupBlob.sha256.in_(gone[at:at + 1000]),
|
||||
))
|
||||
kept = {r.sha256 for r in rows} - set(gone)
|
||||
|
||||
found = []
|
||||
for r in session.execute(text(f"""
|
||||
SELECT DISTINCT ON (sha256) sha256, path, size_bytes, mime
|
||||
FROM ({_LIBRARY}) AS lib ORDER BY sha256, path
|
||||
""")).all():
|
||||
if r.sha256 in kept:
|
||||
continue
|
||||
blob = Blob(r.sha256, r.path, r.size_bytes, r.mime)
|
||||
key = blob_key(blob.sha256, blob.path)
|
||||
if held.get(key) == blob.size_bytes:
|
||||
found.append((blob, key))
|
||||
for at in range(0, len(found), 1000):
|
||||
record(session, tid, found[at:at + 1000])
|
||||
session.commit()
|
||||
return {"listed": len(held), "dropped": len(gone), "adopted": len(found)}
|
||||
@@ -0,0 +1,198 @@
|
||||
"""Restoring from the off-site bucket: the database from one of its dated
|
||||
dumps, then every library file the restored rows name, fetched back to the
|
||||
path each row records.
|
||||
|
||||
The database comes first because it is the map: each image and attachment row
|
||||
holds its file's path and sha256, which is also the file's name in the bucket
|
||||
(`media/ab/cd/<sha256><ext>`). A file is checked against that sha256 on
|
||||
arrival and only then moved into place, so a restore never leaves a wrong or
|
||||
half-written file in the library. A file already on disk at the right size is
|
||||
left alone, which makes a restore safe to run again and lets it bring back
|
||||
only what a damaged library is missing.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from ..models import BackupBlob, ImportSettings
|
||||
from . import backup_destination
|
||||
from .backup_media import _LIBRARY, blob_key
|
||||
from .backup_s3 import Pacer, S3Error, S3Target, get_file, list_objects
|
||||
|
||||
DUMP_PREFIX = "database/"
|
||||
_DUMP_NAME = re.compile(r"^database/fabledcurator-(\d{8}T\d{6}Z)\.dump$")
|
||||
_ALEMBIC_INI = Path(__file__).resolve().parents[3] / "alembic.ini"
|
||||
_MIGRATE_TIMEOUT_S = 1800
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Dump:
|
||||
key: str
|
||||
ts: str # 20261107T134258Z, UTC
|
||||
size_bytes: int
|
||||
counted: bool = False # its row counts sit beside it, for a test restore
|
||||
|
||||
|
||||
def list_dumps(target: S3Target) -> list[Dump]:
|
||||
"""Every database dump in the bucket, newest first."""
|
||||
found, keys, token = [], set(), None
|
||||
while True:
|
||||
page = list_objects(target, DUMP_PREFIX, 1000, continuation=token)
|
||||
for o in page.objects:
|
||||
keys.add(o.key)
|
||||
if m := _DUMP_NAME.match(o.key):
|
||||
found.append((o.key, m.group(1), o.size))
|
||||
token = page.next_token
|
||||
if not token:
|
||||
break
|
||||
dumps = [Dump(k, ts, size, counted=k.removesuffix(".dump") + ".counts.json" in keys)
|
||||
for k, ts, size in found]
|
||||
return sorted(dumps, key=lambda d: d.ts, reverse=True)
|
||||
|
||||
|
||||
def find_dump(target: S3Target, key: str | None) -> Dump:
|
||||
"""The dump at `key`, or the newest when `key` is None or "latest"."""
|
||||
dumps = list_dumps(target)
|
||||
if not dumps:
|
||||
raise S3Error(f"The {target.bucket} bucket holds no database dumps.")
|
||||
if key in (None, "", "latest"):
|
||||
return dumps[0]
|
||||
for d in dumps:
|
||||
if d.key == key:
|
||||
return d
|
||||
raise S3Error(f"{key} is not among the dumps in the {target.bucket} bucket.")
|
||||
|
||||
|
||||
def fetch_dump(target: S3Target, dump: Dump, into: Path, pacer: Pacer | None) -> Path:
|
||||
"""Download a dump into the folder `into`, checked against its listed
|
||||
size. pg_restore reads it from there."""
|
||||
into.mkdir(parents=True, exist_ok=True)
|
||||
path = into / Path(dump.key).name
|
||||
get_file(target, dump.key, path, size=dump.size_bytes, pacer=pacer)
|
||||
return path
|
||||
|
||||
|
||||
def migrate(db_name: str | None = None) -> None:
|
||||
"""Bring a restored database up to this install's schema. A dump taken by
|
||||
an older FabledCurator gains the migrations it predates; one from a newer
|
||||
FabledCurator fails here, with alembic's reason, because this install
|
||||
doesn't know its schema. `db_name` names another database on the same
|
||||
server to bring up instead of this install's own."""
|
||||
env = {**os.environ, "DB_NAME": db_name} if db_name else None
|
||||
with tempfile.TemporaryFile() as err:
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "-m", "alembic", "-c", str(_ALEMBIC_INI), "upgrade", "head"],
|
||||
cwd=_ALEMBIC_INI.parent, stdout=subprocess.DEVNULL, stderr=err,
|
||||
timeout=_MIGRATE_TIMEOUT_S, env=env,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
err.seek(0)
|
||||
lines = err.read().decode(errors="replace").strip().splitlines()
|
||||
raise S3Error("The restored database couldn't be brought up to this "
|
||||
f"version: {lines[-1] if lines else 'alembic failed'}")
|
||||
|
||||
|
||||
def keep_destination(session, target: S3Target, limit_mbps: int | None) -> None:
|
||||
"""Write this install's destination into the restored settings. The
|
||||
dump's own copy of the secret is sealed with the old server's credential
|
||||
key, which never leaves that server, so it can't be opened here."""
|
||||
settings = ImportSettings.load_sync(session)
|
||||
settings.backup_s3_endpoint = target.endpoint
|
||||
settings.backup_s3_bucket = target.bucket
|
||||
settings.backup_s3_access_key = target.access_key
|
||||
settings.backup_s3_region = target.region
|
||||
settings.backup_s3_secret_sealed = backup_destination.seal(target.secret_key)
|
||||
settings.backup_upload_limit_mbps = limit_mbps
|
||||
session.commit()
|
||||
|
||||
|
||||
# ── files ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LibraryFile:
|
||||
sha256: str
|
||||
paths: tuple[str, ...] # every row's path for this sha, sorted
|
||||
size_bytes: int
|
||||
|
||||
|
||||
def library_files(session, *, after: str = "", limit: int = 200) -> list[LibraryFile]:
|
||||
"""The next `limit` distinct files the library names, in sha order after
|
||||
`after`, each with every path a row gives it."""
|
||||
rows = session.execute(text(f"""
|
||||
SELECT lib.sha256, array_agg(DISTINCT lib.path ORDER BY lib.path) AS paths,
|
||||
max(lib.size_bytes) AS size_bytes
|
||||
FROM ({_LIBRARY}) AS lib
|
||||
WHERE lib.sha256 > :after
|
||||
GROUP BY lib.sha256
|
||||
ORDER BY lib.sha256
|
||||
LIMIT :limit
|
||||
"""), {"after": after, "limit": limit}).all()
|
||||
return [LibraryFile(r.sha256, tuple(r.paths), r.size_bytes) for r in rows]
|
||||
|
||||
|
||||
def sent_keys(session, tid: str, shas: list[str]) -> dict[str, str]:
|
||||
"""The bucket key recorded for each sha that was sent to this destination."""
|
||||
if not shas:
|
||||
return {}
|
||||
return dict(session.execute(
|
||||
select(BackupBlob.sha256, BackupBlob.key)
|
||||
.where(BackupBlob.target == tid, BackupBlob.sha256.in_(shas))
|
||||
).all())
|
||||
|
||||
|
||||
def _resolve(path: str, images_root: Path) -> Path:
|
||||
"""Where a row's path lands on this server. A path outside the library
|
||||
folder is refused: a restore writes only inside it."""
|
||||
p = Path(path)
|
||||
if not p.is_absolute():
|
||||
p = images_root / p
|
||||
root = Path(*images_root.parts)
|
||||
if ".." in p.parts or not (p == root or root in p.parents):
|
||||
raise S3Error(f"{path} is outside {images_root}, so it isn't restored there.")
|
||||
return p
|
||||
|
||||
|
||||
def _present(p: Path, size: int) -> bool:
|
||||
try:
|
||||
return p.is_file() and p.stat().st_size == size
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def restore_file(target: S3Target, f: LibraryFile, key: str | None,
|
||||
images_root: Path, pacer: Pacer | None) -> str:
|
||||
"""Put one file back at every path the library gives it. Returns
|
||||
"present" (nothing was missing), "copied" (a path already on disk
|
||||
supplied the others) or "restored" (fetched from the bucket). Raises
|
||||
ObjectMissing when the bucket doesn't hold it, S3Error or OSError
|
||||
otherwise."""
|
||||
places = [_resolve(p, images_root) for p in f.paths]
|
||||
have = [p for p in places if _present(p, f.size_bytes)]
|
||||
need = [p for p in places if p not in have]
|
||||
if not need:
|
||||
return "present"
|
||||
outcome = "copied"
|
||||
if not have:
|
||||
first = need.pop(0)
|
||||
first.parent.mkdir(parents=True, exist_ok=True)
|
||||
get_file(target, key or blob_key(f.sha256, f.paths[0]), first,
|
||||
sha256=f.sha256, size=f.size_bytes, pacer=pacer)
|
||||
have.append(first)
|
||||
outcome = "restored"
|
||||
for p in need:
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
partial = p.with_name(p.name + ".partial")
|
||||
shutil.copyfile(have[0], partial)
|
||||
partial.replace(p)
|
||||
return outcome
|
||||
|
||||
@@ -0,0 +1,533 @@
|
||||
"""A small S3 client for sending backups off the box: AWS Signature Version 4,
|
||||
path-style addressing (endpoint/bucket/key), and the calls a backup needs —
|
||||
put an object (whole, or in parts when its size isn't known up front), read
|
||||
one back, list a prefix.
|
||||
|
||||
FabledCurator puts, lists and reads; it never deletes. The bucket keeps the
|
||||
history and thins it on its own policy, so a key that may only add and read
|
||||
objects is enough, and is the safer key to hand a service: a stolen one cannot
|
||||
wipe the history it writes into.
|
||||
|
||||
Built on the standard library's http.client rather than boto3: three calls
|
||||
don't justify a large dependency, and signing is a pure function the tests
|
||||
hold to AWS's own published examples. Every call is synchronous, because the
|
||||
uploads run in Celery workers; the API runs the Test destination call in a
|
||||
thread.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import http.client
|
||||
import os
|
||||
import socket
|
||||
import ssl
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit
|
||||
|
||||
EMPTY_SHA256 = hashlib.sha256(b"").hexdigest()
|
||||
TIMEOUT_SECONDS = 30.0
|
||||
# A single object can be a multi-gigabyte video or archive; a slow link still
|
||||
# gets this long between bytes before the call gives up.
|
||||
TRANSFER_TIMEOUT_SECONDS = 15 * 60
|
||||
CHUNK = 1 << 20
|
||||
|
||||
|
||||
class S3Error(Exception):
|
||||
"""Why a call to the bucket failed, said to the person."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class S3Target:
|
||||
endpoint: str # http(s)://host[:port][/prefix], no bucket
|
||||
bucket: str
|
||||
access_key: str
|
||||
secret_key: str
|
||||
region: str = "us-east-1"
|
||||
verify_tls: bool = True
|
||||
|
||||
def url(self, key: str = "") -> str:
|
||||
base = f"{self.endpoint.rstrip('/')}/{quote(self.bucket, safe='')}"
|
||||
return f"{base}/{quote(key, safe='/-_.~')}" if key else f"{base}/"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StoredObject:
|
||||
key: str
|
||||
size: int
|
||||
last_modified: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ListPage:
|
||||
objects: list[StoredObject]
|
||||
next_token: str | None # pass back as `continuation` for the next page
|
||||
|
||||
|
||||
# ── Signing ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def _encode(text: str, keep_slash: bool = False) -> str:
|
||||
return quote(text, safe="-_.~" + ("/" if keep_slash else ""))
|
||||
|
||||
|
||||
def _hmac(key: bytes, msg: str) -> bytes:
|
||||
return hmac.new(key, msg.encode(), hashlib.sha256).digest()
|
||||
|
||||
|
||||
def sign(method: str, url: str, headers: dict[str, str], payload_sha256: str,
|
||||
access_key: str, secret_key: str, region: str, at: datetime,
|
||||
service: str = "s3") -> dict[str, str]:
|
||||
"""The request's headers with Host, x-amz-date, x-amz-content-sha256 and
|
||||
Authorization added. Every header given is signed, Host always."""
|
||||
u = urlsplit(url)
|
||||
stamp = at.astimezone(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
day = stamp[:8]
|
||||
out = dict(headers)
|
||||
out["Host"] = u.netloc
|
||||
out["x-amz-date"] = stamp
|
||||
out["x-amz-content-sha256"] = payload_sha256
|
||||
signed = {k.lower(): " ".join(str(v).split()) for k, v in out.items()}
|
||||
names = sorted(signed)
|
||||
|
||||
query = sorted(
|
||||
(_encode(k), _encode(v)) for k, v in parse_qsl(u.query, keep_blank_values=True)
|
||||
)
|
||||
canonical = "\n".join([
|
||||
method.upper(),
|
||||
_encode(unquote(u.path) or "/", keep_slash=True),
|
||||
"&".join(f"{k}={v}" for k, v in query),
|
||||
"".join(f"{n}:{signed[n]}\n" for n in names),
|
||||
";".join(names),
|
||||
payload_sha256,
|
||||
])
|
||||
scope = f"{day}/{region}/{service}/aws4_request"
|
||||
to_sign = "\n".join(["AWS4-HMAC-SHA256", stamp, scope,
|
||||
hashlib.sha256(canonical.encode()).hexdigest()])
|
||||
key = _hmac(_hmac(_hmac(_hmac(f"AWS4{secret_key}".encode(), day), region),
|
||||
service), "aws4_request")
|
||||
signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest()
|
||||
out["Authorization"] = (f"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, "
|
||||
f"SignedHeaders={';'.join(names)}, Signature={signature}")
|
||||
return out
|
||||
|
||||
|
||||
# ── Transport ────────────────────────────────────────────────────────────────
|
||||
|
||||
def _connect(target: S3Target, url: str, timeout: float) -> http.client.HTTPConnection:
|
||||
u = urlsplit(url)
|
||||
if u.scheme == "https":
|
||||
ctx = ssl.create_default_context()
|
||||
if not target.verify_tls:
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
return http.client.HTTPSConnection(
|
||||
u.hostname, u.port, timeout=timeout, context=ctx, blocksize=CHUNK,
|
||||
)
|
||||
if u.scheme != "http":
|
||||
raise S3Error(f"{target.endpoint} isn't an http:// or https:// address.")
|
||||
return http.client.HTTPConnection(u.hostname, u.port, timeout=timeout, blocksize=CHUNK)
|
||||
|
||||
|
||||
def _request_target(url: str) -> str:
|
||||
u = urlsplit(url)
|
||||
return (u.path or "/") + (f"?{u.query}" if u.query else "")
|
||||
|
||||
|
||||
def _reach_error(target: S3Target, exc: Exception) -> S3Error:
|
||||
if isinstance(exc, TimeoutError):
|
||||
return S3Error(f"{target.endpoint} didn't answer in time.")
|
||||
if isinstance(exc, socket.gaierror):
|
||||
return S3Error(f"Couldn't reach {target.endpoint}: the host name doesn't resolve.")
|
||||
if isinstance(exc, ConnectionRefusedError):
|
||||
return S3Error(
|
||||
f"Couldn't reach {target.endpoint}: nothing is listening there "
|
||||
"(check the port)."
|
||||
)
|
||||
if isinstance(exc, ssl.SSLError):
|
||||
return S3Error(f"Couldn't reach {target.endpoint}: TLS failed ({exc.reason}).")
|
||||
return S3Error(f"Couldn't reach {target.endpoint}: {exc}.")
|
||||
|
||||
|
||||
def _says(status: int, reason: str, content: bytes) -> str:
|
||||
"""An S3 error response in words: its Code and Message when it has them."""
|
||||
try:
|
||||
root = ET.fromstring(content) if content else None
|
||||
code = (root.findtext("Code") or "") if root is not None else ""
|
||||
message = (root.findtext("Message") or "") if root is not None else ""
|
||||
except ET.ParseError:
|
||||
code, message = "", ""
|
||||
hints = {
|
||||
"SignatureDoesNotMatch": "the secret key is wrong",
|
||||
"InvalidAccessKeyId": "the access key isn't known there",
|
||||
"NoSuchBucket": "there is no such bucket",
|
||||
"AccessDenied": "the key isn't allowed to do that",
|
||||
"XAmzContentSHA256Mismatch": "the file's bytes don't match its recorded checksum",
|
||||
}
|
||||
said = hints.get(code) or message or reason
|
||||
if not code and 400 <= status < 500:
|
||||
# S3 refuses with an <Error><Code> body. A 4xx without one came from
|
||||
# something else at that address — most often a web server on the
|
||||
# default port, when the S3 service listens on a port of its own.
|
||||
said = ("the address answered, but not as S3 — check the scheme and "
|
||||
"the port (an S3 service usually listens on its own port)")
|
||||
return f"HTTP {status}{f' {code}' if code else ''}: {said}."
|
||||
|
||||
|
||||
# ── Calls ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def file_sha256(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
while chunk := f.read(CHUNK):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def put_file(target: S3Target, key: str, path: Path, *,
|
||||
content_type: str = "application/octet-stream",
|
||||
payload_sha256: str | None = None,
|
||||
pacer: Pacer | None = None,
|
||||
now: datetime | None = None) -> None:
|
||||
"""Upload one file as one object, streamed from disk. `payload_sha256` is
|
||||
the file's sha256 when the caller already knows it (the library stores
|
||||
every file's), which saves reading the file twice; the bucket checks the
|
||||
bytes against it, so a file that changed on disk is refused rather than
|
||||
stored under the wrong name."""
|
||||
digest = payload_sha256 or file_sha256(path)
|
||||
size = path.stat().st_size
|
||||
url = target.url(key)
|
||||
headers = sign("PUT", url, {"Content-Length": str(size), "Content-Type": content_type},
|
||||
digest, target.access_key, target.secret_key, target.region,
|
||||
now or datetime.now(UTC))
|
||||
conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS)
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
conn.request("PUT", _request_target(url),
|
||||
body=pacer.file_chunks(f) if pacer else f, headers=headers)
|
||||
response = conn.getresponse()
|
||||
content = response.read()
|
||||
except (OSError, http.client.HTTPException) as exc:
|
||||
raise _reach_error(target, exc) from None
|
||||
finally:
|
||||
conn.close()
|
||||
if response.status // 100 != 2:
|
||||
raise S3Error(f"Upload of {key} refused: "
|
||||
f"{_says(response.status, response.reason, content)}")
|
||||
|
||||
|
||||
class ObjectMissing(S3Error):
|
||||
"""The bucket holds nothing at the key asked for."""
|
||||
|
||||
|
||||
def get_file(target: S3Target, key: str, path: Path, *,
|
||||
sha256: str | None = None, size: int | None = None,
|
||||
pacer: Pacer | None = None,
|
||||
now: datetime | None = None) -> None:
|
||||
"""Download one object to `path`, streamed to disk. The bytes land under a
|
||||
`.partial` name and are renamed into place when complete, so a broken
|
||||
transfer never leaves a truncated file at `path`. With `sha256` (and
|
||||
`size`) given, the bytes are checked on arrival and a copy that doesn't
|
||||
match is thrown away and reported. Raises ObjectMissing when the bucket
|
||||
has no such object."""
|
||||
url = target.url(key)
|
||||
headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
|
||||
target.region, now or datetime.now(UTC))
|
||||
partial = path.with_name(path.name + ".partial")
|
||||
digest = hashlib.sha256()
|
||||
got = 0
|
||||
conn = _connect(target, url, TRANSFER_TIMEOUT_SECONDS)
|
||||
try:
|
||||
conn.request("GET", _request_target(url), headers=headers)
|
||||
response = conn.getresponse()
|
||||
if response.status == 404:
|
||||
response.read()
|
||||
raise ObjectMissing(f"{key} is not in the {target.bucket} bucket.")
|
||||
if response.status != 200:
|
||||
raise S3Error(f"Reading {key} refused: "
|
||||
f"{_says(response.status, response.reason, response.read())}")
|
||||
with open(partial, "wb") as f:
|
||||
while chunk := response.read(CHUNK):
|
||||
if pacer:
|
||||
pacer.take(len(chunk))
|
||||
digest.update(chunk)
|
||||
got += len(chunk)
|
||||
f.write(chunk)
|
||||
if size is not None and got != size:
|
||||
raise S3Error(f"{key} arrived as {got} bytes, not the {size} expected.")
|
||||
if sha256 and digest.hexdigest() != sha256:
|
||||
raise S3Error(f"{key} arrived with a different fingerprint "
|
||||
f"({digest.hexdigest()[:12]}…), so it wasn't kept.")
|
||||
os.replace(partial, path)
|
||||
except (OSError, http.client.HTTPException) as exc:
|
||||
partial.unlink(missing_ok=True)
|
||||
raise _reach_error(target, exc) from None
|
||||
except S3Error:
|
||||
partial.unlink(missing_ok=True)
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def list_objects(target: S3Target, prefix: str = "", limit: int = 1000, *,
|
||||
continuation: str | None = None,
|
||||
now: datetime | None = None) -> ListPage:
|
||||
"""One page (up to `limit`) of the objects under `prefix`, ListObjectsV2."""
|
||||
params = {"list-type": "2", "max-keys": str(limit)}
|
||||
if prefix:
|
||||
params["prefix"] = prefix
|
||||
if continuation:
|
||||
params["continuation-token"] = continuation
|
||||
url = f"{target.url()}?{urlencode(params, quote_via=quote)}"
|
||||
headers = sign("GET", url, {}, EMPTY_SHA256, target.access_key, target.secret_key,
|
||||
target.region, now or datetime.now(UTC))
|
||||
conn = _connect(target, url, TIMEOUT_SECONDS)
|
||||
try:
|
||||
conn.request("GET", _request_target(url), headers=headers)
|
||||
response = conn.getresponse()
|
||||
content = response.read()
|
||||
except (OSError, http.client.HTTPException) as exc:
|
||||
raise _reach_error(target, exc) from None
|
||||
finally:
|
||||
conn.close()
|
||||
if response.status != 200:
|
||||
raise S3Error(f"Listing {target.bucket} refused: "
|
||||
f"{_says(response.status, response.reason, content)}")
|
||||
try:
|
||||
root = ET.fromstring(content)
|
||||
except ET.ParseError:
|
||||
raise S3Error(
|
||||
f"{target.endpoint} answered a listing with something that isn't S3's XML."
|
||||
) from None
|
||||
objects, next_token = [], None
|
||||
for item in root.iter():
|
||||
tag = item.tag.rsplit("}", 1)[-1]
|
||||
if tag == "NextContinuationToken":
|
||||
next_token = item.text or None
|
||||
if tag != "Contents":
|
||||
continue
|
||||
fields = {child.tag.rsplit("}", 1)[-1]: (child.text or "") for child in item}
|
||||
objects.append(StoredObject(fields.get("Key", ""), int(fields.get("Size") or 0),
|
||||
fields.get("LastModified", "")))
|
||||
return ListPage(objects, next_token)
|
||||
|
||||
|
||||
# ── Multipart: one object sent in parts ─────────────────────────────────────
|
||||
|
||||
# Each part is held in memory while it is sent, so this bounds the memory a
|
||||
# streamed upload uses. S3 allows 10,000 parts, which at this size is 640 GiB
|
||||
# for one object, and every part but the last must be at least 5 MiB.
|
||||
PART_SIZE = 64 * CHUNK
|
||||
# A part the bucket couldn't take (it didn't answer, or said it was busy) is
|
||||
# sent again after a rest that grows each time: 15 s, then 60 s.
|
||||
PART_RETRY_WAITS = (15, 60)
|
||||
_BUSY = frozenset({429, 500, 502, 503, 504})
|
||||
_MAX_RETRY_AFTER = 300
|
||||
|
||||
|
||||
class Pacer:
|
||||
"""Holds the bytes a job moves to an average rate, across every request
|
||||
it makes, so a backup or a restore leaves room on the link for everything
|
||||
else."""
|
||||
|
||||
def __init__(self, mbps: int | None):
|
||||
self.rate = mbps * 1_000_000 / 8 if mbps else None # bytes per second
|
||||
self.sent = 0
|
||||
self.started = time.monotonic()
|
||||
|
||||
def take(self, n: int) -> None:
|
||||
"""Count `n` bytes moved, pausing while the job is ahead of its rate."""
|
||||
if self.rate:
|
||||
self.sent += n
|
||||
ahead = self.sent / self.rate - (time.monotonic() - self.started)
|
||||
if ahead > 0:
|
||||
time.sleep(ahead)
|
||||
|
||||
def chunks(self, data: bytes) -> Iterator[memoryview]:
|
||||
view = memoryview(data)
|
||||
for at in range(0, len(view), CHUNK):
|
||||
piece = view[at:at + CHUNK]
|
||||
self.take(len(piece))
|
||||
yield piece
|
||||
|
||||
def file_chunks(self, f) -> Iterator[bytes]:
|
||||
while piece := f.read(CHUNK):
|
||||
self.take(len(piece))
|
||||
yield piece
|
||||
|
||||
|
||||
def _send(target: S3Target, method: str, url: str, body: bytes = b"",
|
||||
headers: dict[str, str] | None = None, *, timeout: float = TIMEOUT_SECONDS,
|
||||
pacer: Pacer | None = None,
|
||||
now: datetime | None = None) -> tuple[int, str, http.client.HTTPMessage, bytes]:
|
||||
"""Sign and send one request with an in-memory body, paced when a pacer
|
||||
is given. Returns (status, reason, headers, content)."""
|
||||
digest = hashlib.sha256(body).hexdigest()
|
||||
signed = sign(method, url, {"Content-Length": str(len(body)), **(headers or {})},
|
||||
digest, target.access_key, target.secret_key, target.region,
|
||||
now or datetime.now(UTC))
|
||||
conn = _connect(target, url, timeout)
|
||||
try:
|
||||
conn.request(method, _request_target(url),
|
||||
body=pacer.chunks(body) if pacer and body else body, headers=signed)
|
||||
response = conn.getresponse()
|
||||
content = response.read()
|
||||
except (OSError, http.client.HTTPException) as exc:
|
||||
raise _reach_error(target, exc) from None
|
||||
finally:
|
||||
conn.close()
|
||||
return response.status, response.reason, response.headers, content
|
||||
|
||||
|
||||
def _xml_field(content: bytes, name: str) -> str | None:
|
||||
try:
|
||||
root = ET.fromstring(content)
|
||||
except ET.ParseError:
|
||||
return None
|
||||
for item in root.iter():
|
||||
if item.tag.rsplit("}", 1)[-1] == name:
|
||||
return item.text
|
||||
return None
|
||||
|
||||
|
||||
def _is_error_body(content: bytes) -> bool:
|
||||
try:
|
||||
return bool(content) and ET.fromstring(content).tag.rsplit("}", 1)[-1] == "Error"
|
||||
except ET.ParseError:
|
||||
return False
|
||||
|
||||
|
||||
def _retry_after(headers: http.client.HTTPMessage | None, default: int) -> int:
|
||||
try:
|
||||
asked = int((headers or {}).get("Retry-After") or default)
|
||||
except ValueError:
|
||||
asked = default
|
||||
return max(0, min(asked, _MAX_RETRY_AFTER))
|
||||
|
||||
|
||||
class Upload:
|
||||
"""One object written as a series of parts, for data whose size isn't known
|
||||
when it starts, such as a database dump streamed from pg_dump.
|
||||
|
||||
The object appears in the bucket only when `complete()` succeeds. Until
|
||||
then the parts are invisible, so a stream that fails partway leaves
|
||||
nothing behind that looks like a finished backup. `abort()` asks the
|
||||
bucket to drop the parts at once; if the abort is refused or never
|
||||
arrives, S3 services expire an unfinished upload on their own (Fabled
|
||||
Archives after a day)."""
|
||||
|
||||
def __init__(self, target: S3Target, key: str, *,
|
||||
content_type: str = "application/octet-stream",
|
||||
pacer: Pacer | None = None):
|
||||
self.target = target
|
||||
self.key = key
|
||||
self.size = 0
|
||||
self.pacer = pacer
|
||||
self._etags: list[str] = []
|
||||
status, reason, _, content = _send(
|
||||
target, "POST", f"{target.url(key)}?uploads", headers={"Content-Type": content_type},
|
||||
)
|
||||
upload_id = _xml_field(content, "UploadId") if status == 200 else None
|
||||
if not upload_id:
|
||||
raise S3Error(f"Starting the upload of {key} refused: "
|
||||
f"{_says(status, reason, content)}")
|
||||
self.upload_id = upload_id
|
||||
|
||||
def _url(self, **params: str) -> str:
|
||||
query = urlencode({**params, "uploadId": self.upload_id}, quote_via=quote)
|
||||
return f"{self.target.url(self.key)}?{query}"
|
||||
|
||||
@property
|
||||
def parts(self) -> int:
|
||||
return len(self._etags)
|
||||
|
||||
def add(self, data: bytes) -> None:
|
||||
"""Send the next part, resting and sending it again when the bucket
|
||||
doesn't answer or says it is busy."""
|
||||
n = len(self._etags) + 1
|
||||
for wait in (*PART_RETRY_WAITS, None):
|
||||
headers = None
|
||||
try:
|
||||
status, reason, headers, content = _send(
|
||||
self.target, "PUT", self._url(partNumber=str(n)), data,
|
||||
timeout=TRANSFER_TIMEOUT_SECONDS, pacer=self.pacer,
|
||||
)
|
||||
except S3Error as exc:
|
||||
if wait is None:
|
||||
raise S3Error(f"Part {n} of {self.key}: {exc}") from None
|
||||
time.sleep(wait)
|
||||
continue
|
||||
etag = headers.get("ETag") if status == 200 else None
|
||||
if etag:
|
||||
self._etags.append(etag)
|
||||
self.size += len(data)
|
||||
return
|
||||
if status not in _BUSY or wait is None:
|
||||
raise S3Error(f"Part {n} of {self.key} refused: "
|
||||
f"{_says(status, reason, content)}")
|
||||
time.sleep(_retry_after(headers, wait))
|
||||
|
||||
def complete(self) -> None:
|
||||
"""Join the parts into the object. S3 can refuse a completion with an
|
||||
error body under a 200 status, so the body is read too."""
|
||||
if not self._etags:
|
||||
raise S3Error(f"Nothing was sent for {self.key}.")
|
||||
body = "<CompleteMultipartUpload>" + "".join(
|
||||
f"<Part><PartNumber>{n}</PartNumber><ETag>{etag}</ETag></Part>"
|
||||
for n, etag in enumerate(self._etags, start=1)
|
||||
) + "</CompleteMultipartUpload>"
|
||||
status, reason, _, content = _send(
|
||||
self.target, "POST", self._url(), body.encode(),
|
||||
headers={"Content-Type": "application/xml"}, timeout=TRANSFER_TIMEOUT_SECONDS,
|
||||
)
|
||||
if status != 200 or _is_error_body(content):
|
||||
raise S3Error(f"Finishing the upload of {self.key} refused: "
|
||||
f"{_says(status, reason, content)}")
|
||||
|
||||
def abort(self) -> None:
|
||||
"""Drop the parts sent so far. Never raises: it runs while another
|
||||
failure is already being reported, and the bucket expires the parts on
|
||||
its own if this doesn't reach it."""
|
||||
try:
|
||||
_send(self.target, "DELETE", self._url())
|
||||
except Exception: # noqa: BLE001 — best effort, see above
|
||||
pass
|
||||
|
||||
|
||||
def put_file_in_parts(target: S3Target, key: str, path: Path, *, sha256: str,
|
||||
content_type: str = "application/octet-stream",
|
||||
pacer: Pacer | None = None) -> int:
|
||||
"""Upload a large file as a multipart object, read from disk a part at a
|
||||
time. The whole file's sha256 is computed on the way and must equal
|
||||
`sha256` before the upload is completed; otherwise it is aborted, so a file
|
||||
that changed on disk is never stored under its old name. Returns the
|
||||
number of bytes sent."""
|
||||
upload = Upload(target, key, content_type=content_type, pacer=pacer)
|
||||
digest = hashlib.sha256()
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
while part := f.read(PART_SIZE):
|
||||
digest.update(part)
|
||||
upload.add(part)
|
||||
if digest.hexdigest() != sha256:
|
||||
raise S3Error(f"{path} no longer matches its recorded checksum; not stored.")
|
||||
upload.complete()
|
||||
except BaseException:
|
||||
upload.abort()
|
||||
raise
|
||||
return upload.size
|
||||
|
||||
|
||||
def object_present(target: S3Target, key: str, size: int) -> bool:
|
||||
"""Whether `key` is already in the bucket at `size` bytes. Used when a put
|
||||
is refused: an append-only bucket refuses to replace an object that is
|
||||
already there, which for a content-addressed key means the work is done."""
|
||||
try:
|
||||
page = list_objects(target, prefix=key, limit=1)
|
||||
except S3Error:
|
||||
return False
|
||||
return any(o.key == key and o.size == size for o in page.objects)
|
||||
@@ -1,12 +1,11 @@
|
||||
"""FC-3h: first-class backup/restore service for FC.
|
||||
|
||||
Two independent backup kinds:
|
||||
- 'db' — pg_dump only; fast; nightly via Beat (settings-gated)
|
||||
- 'images' — tar+zstd of /images; slow; manual trigger only
|
||||
|
||||
Files live under <images_root>/_backups/. Each backup writes:
|
||||
fc_<kind>_<ts>.{sql|tar.zst} — the artifact
|
||||
fc_<kind>_<ts>.json — manifest (kind/tag/triggered_by)
|
||||
The database dump (pg_dump), nightly via Beat when the setting is on. With an
|
||||
off-site destination set up, the dump streams straight to that bucket
|
||||
(backup_db_offsite), and images and attachments go there too (backup_media).
|
||||
Without one, the dump is written under <images_root>/_backups/:
|
||||
fc_db_<ts>.dump — the dump (custom format)
|
||||
fc_db_<ts>.json — manifest (tag, trigger, row counts)
|
||||
|
||||
Service functions are sync (subprocess-bound). Celery tasks in
|
||||
backend.app.tasks.backup wrap each one with task_run-tracked
|
||||
@@ -15,33 +14,19 @@ lifecycle + soft/hard time limits + retention bookkeeping.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
_BACKUPS_DIRNAME = "_backups"
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Excluded from the images tarball, and each for its own reason (#4233, #4234):
|
||||
#
|
||||
# _backups — the archive would otherwise contain every previous archive.
|
||||
# This is not hypothetical: the 2026-05-23/24 runs, taken before
|
||||
# this exclude existed, grew 43G -> 107G -> ... -> 2123G as each
|
||||
# swallowed its predecessors, and cost 4.3T of the images
|
||||
# filesystem until they were reclaimed on 2026-09-21.
|
||||
# _quarantine — holds files deliberately pulled OUT of the library.
|
||||
# secrets — `credential_key.b64`, the key that decrypts the stored
|
||||
# Patreon/SubscribeStar session credentials.
|
||||
# cookies — those session cookies themselves.
|
||||
#
|
||||
# The last two are the ones worth stating plainly: an images tarball is a media
|
||||
# archive, and a media archive that carries the key to the operator's accounts
|
||||
# is a credential leak wearing a backup's name. Encryption at rest buys nothing
|
||||
# when the key rides along in the same file. A restore therefore does NOT
|
||||
# re-establish credentials — you sign in again, which is the correct outcome.
|
||||
_IMAGES_EXCLUDED_DIRNAMES = ("_backups", "_quarantine", "secrets", "cookies")
|
||||
_BACKUPS_DIRNAME = "_backups"
|
||||
|
||||
# Subprocess-level guardrails BEYOND the Celery soft_time_limit. The Celery
|
||||
# soft limit signals the Python process; subprocess.Popen in a blocking syscall
|
||||
@@ -51,9 +36,7 @@ _IMAGES_EXCLUDED_DIRNAMES = ("_backups", "_quarantine", "secrets", "cookies")
|
||||
# matters because an NFS D-state hang defeats even Celery's SIGKILL (the failure
|
||||
# that wedged the maintenance lane for hours, #739).
|
||||
# backup_db_task: soft=1800s / hard=2100s → 1700s
|
||||
# backup_images_task: soft=21600s / hard=23400s → 21000s
|
||||
_DB_SUBPROCESS_TIMEOUT_S = 1700 # ~28 min, under the 30-min DB soft limit
|
||||
_IMAGES_SUBPROCESS_TIMEOUT_S = 21000 # ~5.8 hr, under the 6-hr images soft limit
|
||||
# Grace after SIGKILL to reap the child. If it can't be reaped in this window
|
||||
# (an uninterruptible NFS D-state — the failure mode that wedged the
|
||||
# concurrency-1 maintenance lane for hours, operator-flagged 2026-06-07), we
|
||||
@@ -62,6 +45,85 @@ _IMAGES_SUBPROCESS_TIMEOUT_S = 21000 # ~5.8 hr, under the 6-hr images soft
|
||||
_KILL_REAP_GRACE_S = 10
|
||||
|
||||
|
||||
# Tables whose rows stay out of every dump. `credential` holds account logins,
|
||||
# sealed with the key in /images/secrets: a restored install keeps the table,
|
||||
# empty, and you sign in to each account again.
|
||||
DATA_LEFT_OUT = ("credential",)
|
||||
|
||||
# Every database dump, local or off-site: custom format (-Fc, compressed and
|
||||
# restored with pg_restore), without ownership or grants so it loads into any
|
||||
# install.
|
||||
_PG_DUMP = ["pg_dump", "--no-owner", "--no-acl", "-Fc",
|
||||
*(f"--exclude-table-data={t}" for t in DATA_LEFT_OUT)]
|
||||
|
||||
|
||||
def table_counts(conn) -> dict[str, int]:
|
||||
"""How many rows each table in the public schema holds, read on a psycopg
|
||||
connection, in the order of the tables' names."""
|
||||
from psycopg import sql
|
||||
|
||||
names = [r[0] for r in conn.execute(
|
||||
"SELECT tablename FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename")]
|
||||
return {
|
||||
n: conn.execute(sql.SQL("SELECT count(*) FROM {}").format(sql.Identifier(n))).fetchone()[0]
|
||||
for n in names
|
||||
}
|
||||
|
||||
|
||||
def database_counts(conn) -> dict:
|
||||
"""{"alembic": the schema's revision, "tables": {name: rows}}."""
|
||||
has_version = conn.execute("SELECT to_regclass('public.alembic_version')").fetchone()[0]
|
||||
alembic = (conn.execute("SELECT version_num FROM alembic_version").fetchone()
|
||||
if has_version else None)
|
||||
return {"alembic": alembic[0] if alembic else None, "tables": table_counts(conn)}
|
||||
|
||||
|
||||
def read_counts(db_url: str) -> dict:
|
||||
"""database_counts for the database at `db_url`, as it is now."""
|
||||
import psycopg
|
||||
|
||||
with psycopg.connect(_libpq_url(db_url)) as conn:
|
||||
return database_counts(conn)
|
||||
|
||||
|
||||
class DumpSnapshot:
|
||||
"""A read-only transaction whose snapshot pg_dump dumps from (it is handed
|
||||
over with --snapshot), so rows counted here are exactly the rows the dump
|
||||
holds. A test restore checks every table against these counts."""
|
||||
|
||||
def __init__(self, conn):
|
||||
self.conn = conn
|
||||
self.id = conn.execute("SELECT pg_export_snapshot()").fetchone()[0]
|
||||
|
||||
def counts(self) -> dict | None:
|
||||
"""The counts, or None when they can't be read: the dump is good
|
||||
without them, and a test restore then says none were recorded."""
|
||||
try:
|
||||
return self._counts()
|
||||
except Exception: # noqa: BLE001 — see above
|
||||
log.exception("backup: the dump's row counts couldn't be read")
|
||||
return None
|
||||
|
||||
def _counts(self) -> dict:
|
||||
counts = database_counts(self.conn)
|
||||
for t in DATA_LEFT_OUT:
|
||||
if t in counts["tables"]:
|
||||
counts["tables"][t] = 0 # the dump holds the table, not its rows
|
||||
return counts
|
||||
|
||||
|
||||
@contextmanager
|
||||
def dump_snapshot(db_url: str):
|
||||
"""Open the snapshot a dump is taken from. It stays open until the dump
|
||||
is finished: pg_dump can only take it up while it is."""
|
||||
import psycopg
|
||||
|
||||
with psycopg.connect(_libpq_url(db_url)) as conn:
|
||||
conn.isolation_level = psycopg.IsolationLevel.REPEATABLE_READ
|
||||
conn.read_only = True
|
||||
yield DumpSnapshot(conn)
|
||||
|
||||
|
||||
def _run_bounded(cmd: list[str], timeout: int) -> None:
|
||||
"""subprocess.run(check=True, timeout) whose reaper can't itself hang.
|
||||
|
||||
@@ -117,7 +179,7 @@ def _file_size_or_none(path: Path) -> int | None:
|
||||
def _write_manifest(
|
||||
out_dir: Path, *, kind: str, ts: str,
|
||||
tag: str | None, triggered_by: str,
|
||||
artifact_path: Path,
|
||||
artifact_path: Path, counts: dict | None = None,
|
||||
) -> Path:
|
||||
manifest = {
|
||||
"kind": kind,
|
||||
@@ -127,6 +189,8 @@ def _write_manifest(
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
"artifact_path": str(artifact_path),
|
||||
}
|
||||
if counts is not None:
|
||||
manifest["counts"] = counts
|
||||
mf = out_dir / f"fc_{kind}_{ts}.json"
|
||||
mf.write_text(json.dumps(manifest, indent=2))
|
||||
return mf
|
||||
@@ -152,104 +216,215 @@ def backup_db(
|
||||
os.close(fd)
|
||||
tmp_path = Path(tmp_name)
|
||||
try:
|
||||
_run_bounded(
|
||||
[
|
||||
"pg_dump", "--no-owner", "--no-acl", "-Fc",
|
||||
"-f", str(tmp_path), _libpq_url(db_url),
|
||||
],
|
||||
_DB_SUBPROCESS_TIMEOUT_S,
|
||||
)
|
||||
with dump_snapshot(db_url) as snap:
|
||||
_run_bounded(
|
||||
_gently([*_PG_DUMP, f"--snapshot={snap.id}", "-f", str(tmp_path),
|
||||
_libpq_url(db_url)]),
|
||||
_DB_SUBPROCESS_TIMEOUT_S,
|
||||
)
|
||||
counts = snap.counts()
|
||||
shutil.move(str(tmp_path), str(sql_path))
|
||||
finally:
|
||||
if tmp_path.exists():
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
manifest_path = _write_manifest(
|
||||
out_dir, kind="db", ts=ts, tag=tag, triggered_by=triggered_by,
|
||||
artifact_path=sql_path,
|
||||
artifact_path=sql_path, counts=counts,
|
||||
)
|
||||
return {
|
||||
"kind": "db",
|
||||
"ts": ts,
|
||||
"sql_path": str(sql_path),
|
||||
"tar_path": None,
|
||||
"manifest_path": str(manifest_path),
|
||||
"size_bytes": _file_size_or_none(sql_path),
|
||||
"counts": counts,
|
||||
}
|
||||
|
||||
|
||||
def backup_images(
|
||||
*, images_root: Path,
|
||||
tag: str | None = None, triggered_by: str = "manual",
|
||||
) -> dict:
|
||||
"""Run tar --zstd over images_root; write .tar.zst + manifest."""
|
||||
class DumpFailed(Exception):
|
||||
"""pg_dump exited with an error; the message is its last line of output."""
|
||||
|
||||
|
||||
def offsite_dump_key(ts: str) -> str:
|
||||
"""Where a database dump lands in the off-site bucket. Dated names sit
|
||||
side by side under one prefix, which is what lets the bucket's retention
|
||||
thin them by age."""
|
||||
return f"database/fabledcurator-{ts}.dump"
|
||||
|
||||
|
||||
def _gently(cmd: list[str]) -> list[str]:
|
||||
"""`cmd` at low CPU priority, and low disk priority where `ionice` is
|
||||
installed, so a long dump leaves the server responsive."""
|
||||
io = ["ionice", "-c", "2", "-n", "7"] if shutil.which("ionice") else []
|
||||
return ["nice", "-n", "10", *io, *cmd]
|
||||
|
||||
|
||||
def _last_line(f) -> str | None:
|
||||
f.seek(0)
|
||||
lines = f.read().decode(errors="replace").strip().splitlines()
|
||||
return lines[-1].strip() if lines else None
|
||||
|
||||
|
||||
def backup_db_offsite(*, db_url: str, target, limit_mbps: int | None = None) -> dict:
|
||||
"""Stream `pg_dump` straight into the off-site bucket, in parts, with no
|
||||
copy staged on this server. The object appears only once pg_dump has
|
||||
exited cleanly and every part has arrived; a dump that fails partway
|
||||
leaves nothing in the bucket, and raises DumpFailed with pg_dump's own
|
||||
last line. Returns a dict for the caller to record in BackupRun, with
|
||||
`counts`: each table's rows in the dump, for the caller to store beside
|
||||
it."""
|
||||
ts = _now_ts()
|
||||
out_dir = _backups_dir(images_root)
|
||||
tar_path = out_dir / f"fc_images_{ts}.tar.zst"
|
||||
# No local-temp here (the archive is hundreds of GB — it can't stage in
|
||||
# /tmp), but bounded-kill still applies so a tar wedged on NFS fails fast
|
||||
# rather than holding the lane for hours.
|
||||
_run_bounded(
|
||||
[
|
||||
"tar", "--zstd", "-cf", str(tar_path),
|
||||
"-C", str(images_root.parent), images_root.name,
|
||||
*(
|
||||
f"--exclude={images_root.name}/{name}"
|
||||
for name in _IMAGES_EXCLUDED_DIRNAMES
|
||||
),
|
||||
],
|
||||
_IMAGES_SUBPROCESS_TIMEOUT_S,
|
||||
)
|
||||
manifest_path = _write_manifest(
|
||||
out_dir, kind="images", ts=ts, tag=tag, triggered_by=triggered_by,
|
||||
artifact_path=tar_path,
|
||||
)
|
||||
with dump_snapshot(db_url) as snap:
|
||||
result = _stream_dump(db_url, target, ts, limit_mbps, snap.id)
|
||||
result["counts"] = snap.counts()
|
||||
return result
|
||||
|
||||
|
||||
def _stream_dump(db_url: str, target, ts: str, limit_mbps: int | None, snapshot: str) -> dict:
|
||||
from .backup_s3 import PART_SIZE, Pacer, Upload
|
||||
|
||||
key = offsite_dump_key(ts)
|
||||
deadline = time.monotonic() + _DB_SUBPROCESS_TIMEOUT_S
|
||||
with tempfile.TemporaryFile() as err:
|
||||
proc = subprocess.Popen(
|
||||
_gently([*_PG_DUMP, f"--snapshot={snapshot}", _libpq_url(db_url)]),
|
||||
stdout=subprocess.PIPE, stderr=err,
|
||||
)
|
||||
upload = None
|
||||
try:
|
||||
upload = Upload(target, key, pacer=Pacer(limit_mbps))
|
||||
while part := proc.stdout.read(PART_SIZE):
|
||||
if time.monotonic() > deadline:
|
||||
raise subprocess.TimeoutExpired(proc.args, _DB_SUBPROCESS_TIMEOUT_S)
|
||||
upload.add(part)
|
||||
proc.stdout.close()
|
||||
proc.wait(timeout=_KILL_REAP_GRACE_S * 6)
|
||||
if proc.returncode != 0:
|
||||
raise DumpFailed(_last_line(err) or f"pg_dump exited with {proc.returncode}")
|
||||
upload.complete()
|
||||
except BaseException:
|
||||
if proc.poll() is None:
|
||||
proc.kill()
|
||||
try:
|
||||
proc.wait(timeout=_KILL_REAP_GRACE_S)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass # unkillable (D-state): abandon the reap, fail fast
|
||||
if upload is not None:
|
||||
upload.abort()
|
||||
raise
|
||||
return {
|
||||
"kind": "images",
|
||||
"kind": "db",
|
||||
"ts": ts,
|
||||
"sql_path": None,
|
||||
"tar_path": str(tar_path),
|
||||
"manifest_path": str(manifest_path),
|
||||
"size_bytes": _file_size_or_none(tar_path),
|
||||
"key": key,
|
||||
"bucket": target.bucket,
|
||||
"endpoint": target.endpoint,
|
||||
"parts": upload.parts,
|
||||
"size_bytes": upload.size,
|
||||
}
|
||||
|
||||
|
||||
class RestoreFailed(Exception):
|
||||
"""psql or pg_restore exited with an error; the message is its last line."""
|
||||
|
||||
|
||||
# A restore waits this long for other connections to let go of the tables it
|
||||
# replaces, then gives up with Postgres' own reason instead of queueing every
|
||||
# other query behind it.
|
||||
_RESTORE_LOCK_TIMEOUT = "60s"
|
||||
# pg_restore of a large library (vector tables, rebuilt indexes) runs well past
|
||||
# a dump's length; it is bounded on its own.
|
||||
_RESTORE_SUBPROCESS_TIMEOUT_S = 3 * 3600
|
||||
|
||||
|
||||
def _run_reporting(cmd: list[str], timeout: int, env: dict | None = None) -> None:
|
||||
"""Run `cmd`; raise RestoreFailed with the last line it printed when it
|
||||
exits non-zero."""
|
||||
with tempfile.TemporaryFile() as err:
|
||||
proc = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=err, timeout=timeout,
|
||||
env=env)
|
||||
if proc.returncode != 0:
|
||||
raise RestoreFailed(_last_line(err) or f"{cmd[0]} exited with {proc.returncode}")
|
||||
|
||||
|
||||
# Every other connection to this database: the web server's, the other
|
||||
# workers'. Ended before the tables are dropped, so none holds a lock the drop
|
||||
# waits on, and again once the restore is in, so none keeps statements it
|
||||
# prepared against the old tables. Each reconnects on its next query.
|
||||
_END_OTHER_CONNECTIONS = (
|
||||
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
|
||||
"WHERE datname = current_database() AND pid <> pg_backend_pid();"
|
||||
)
|
||||
|
||||
|
||||
def end_other_connections(*, db_url: str) -> None:
|
||||
_run_reporting(["psql", _libpq_url(db_url), "-v", "ON_ERROR_STOP=1",
|
||||
"-c", _END_OTHER_CONNECTIONS], 60)
|
||||
|
||||
|
||||
def wipe_db(*, db_url: str) -> None:
|
||||
"""Drop and recreate the public schema: an empty database, ready for a
|
||||
dump or for `alembic upgrade head`."""
|
||||
_run_reporting(
|
||||
[
|
||||
"psql", _libpq_url(db_url), "-v", "ON_ERROR_STOP=1", "-c",
|
||||
f"{_END_OTHER_CONNECTIONS} DROP SCHEMA IF EXISTS public CASCADE; "
|
||||
"CREATE SCHEMA public;",
|
||||
],
|
||||
120,
|
||||
# Set for the session, not as a SET inside the same -c string.
|
||||
env={**os.environ, "PGOPTIONS": f"-c lock_timeout={_RESTORE_LOCK_TIMEOUT}"},
|
||||
)
|
||||
|
||||
|
||||
# A pg_restore newer than the server sets parameters the server doesn't have
|
||||
# (pg_restore 17 against Postgres 16: `SET transaction_timeout = 0`). The
|
||||
# statement fails and nothing depends on it, so its error is not a failed
|
||||
# restore.
|
||||
_HARMLESS_RESTORE_ERRORS = (
|
||||
'unrecognized configuration parameter "transaction_timeout"',
|
||||
)
|
||||
|
||||
|
||||
def _restore_errors(stderr: str) -> list[str]:
|
||||
"""pg_restore's error lines, less the harmless ones."""
|
||||
return [
|
||||
line.strip() for line in stderr.splitlines()
|
||||
if "error:" in line and not any(h in line for h in _HARMLESS_RESTORE_ERRORS)
|
||||
]
|
||||
|
||||
|
||||
def restore_db(*, db_url: str, sql_path: Path) -> None:
|
||||
"""Wipe public schema, then load from the custom-format dump. Raises on
|
||||
subprocess failure; partial-restore state is the caller's concern."""
|
||||
libpq = _libpq_url(db_url)
|
||||
subprocess.run(
|
||||
[
|
||||
"psql", libpq, "-c",
|
||||
"DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;",
|
||||
],
|
||||
capture_output=True, check=True, timeout=120,
|
||||
)
|
||||
# Custom-format (-Fc) dumps are restored with pg_restore, not psql.
|
||||
subprocess.run(
|
||||
["pg_restore", "--no-owner", "--no-acl", "-d", libpq, str(sql_path)],
|
||||
capture_output=True, check=True,
|
||||
timeout=_DB_SUBPROCESS_TIMEOUT_S,
|
||||
)
|
||||
|
||||
|
||||
def restore_images(*, images_root: Path, tar_path: Path) -> None:
|
||||
"""Untar over images_root.parent. Additive — files NOT in the
|
||||
tarball are NOT removed. Caller wipes first if a clean restore
|
||||
is needed."""
|
||||
subprocess.run(
|
||||
[
|
||||
"tar", "--zstd", "-xf", str(tar_path),
|
||||
"-C", str(images_root.parent),
|
||||
],
|
||||
capture_output=True, check=True,
|
||||
timeout=_IMAGES_SUBPROCESS_TIMEOUT_S,
|
||||
)
|
||||
"""Wipe the public schema, then load the custom-format dump. Any error
|
||||
pg_restore reports, other than a harmless one, raises RestoreFailed with
|
||||
the first; the database is then partly loaded, and the caller decides what
|
||||
to put back."""
|
||||
wipe_db(db_url=db_url)
|
||||
# Custom-format (-Fc) dumps are restored with pg_restore, not psql. It runs
|
||||
# on past errors (it can't skip the harmless one and stop at the rest), so
|
||||
# its errors are judged once it is done.
|
||||
cmd = _gently(["pg_restore", "--no-owner", "--no-acl",
|
||||
"-d", _libpq_url(db_url), str(sql_path)])
|
||||
with tempfile.TemporaryFile() as err:
|
||||
proc = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=err,
|
||||
timeout=_RESTORE_SUBPROCESS_TIMEOUT_S)
|
||||
if proc.returncode == 0:
|
||||
return
|
||||
err.seek(0)
|
||||
said = err.read().decode(errors="replace")
|
||||
errors = _restore_errors(said)
|
||||
if errors:
|
||||
raise RestoreFailed(errors[0] if len(errors) == 1
|
||||
else f"{errors[0]} ({len(errors)} errors in all)")
|
||||
if not any(h in said for h in _HARMLESS_RESTORE_ERRORS):
|
||||
# A failure with no error line to show for it.
|
||||
lines = said.strip().splitlines()
|
||||
raise RestoreFailed(lines[-1].strip() if lines
|
||||
else f"pg_restore exited with {proc.returncode}")
|
||||
|
||||
|
||||
def unlink_artifact_files(
|
||||
*,
|
||||
sql_path: str | None,
|
||||
tar_path: str | None,
|
||||
manifest_path: str | None,
|
||||
) -> dict:
|
||||
"""Best-effort unlink of all on-disk files for a BackupRun row.
|
||||
@@ -258,7 +433,6 @@ def unlink_artifact_files(
|
||||
deleted: dict = {}
|
||||
for label, p in (
|
||||
("sql", sql_path),
|
||||
("tar", tar_path),
|
||||
("manifest", manifest_path),
|
||||
):
|
||||
if not p:
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Testing a database dump without touching this install.
|
||||
|
||||
The dump is loaded into a separate, temporary database on the same Postgres
|
||||
server, using the same steps a real restore takes. Every table is counted
|
||||
there and compared, the copy is brought up to this version, and the copy is
|
||||
then dropped. The live database is only read, to show its counts alongside.
|
||||
|
||||
Every dump records how many rows each table held, counted in the same
|
||||
snapshot pg_dump dumped from, so the comparison is exact. For a dump in the
|
||||
bucket the counts sit beside it as `<dump name>.counts.json`; for a dump on
|
||||
this server they are in its manifest file. A dump taken before counts were
|
||||
recorded can still be tested: it is then compared with the live database,
|
||||
where a difference is what changed after the dump was taken, not a fault.
|
||||
|
||||
The temporary database needs about as much free space on the database's disk
|
||||
as the live one takes, and only for the length of the test.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.engine import make_url
|
||||
|
||||
from . import backup_restore, backup_service
|
||||
from .backup_s3 import ObjectMissing, S3Target, get_file, put_file
|
||||
|
||||
SCRATCH_SUFFIX = "_restore_test"
|
||||
|
||||
|
||||
def counts_key(dump_key: str) -> str:
|
||||
"""Where a dump's row counts sit in the bucket, beside the dump."""
|
||||
return dump_key.removesuffix(".dump") + ".counts.json"
|
||||
|
||||
|
||||
def store_counts(target: S3Target, dump_key: str, counts: dict) -> None:
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = Path(d) / "counts.json"
|
||||
path.write_text(json.dumps(counts))
|
||||
put_file(target, counts_key(dump_key), path, content_type="application/json")
|
||||
|
||||
|
||||
def fetch_counts(target: S3Target, dump_key: str) -> dict | None:
|
||||
"""The counts recorded with a dump in the bucket, or None for a dump that
|
||||
has none."""
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = Path(d) / "counts.json"
|
||||
try:
|
||||
get_file(target, counts_key(dump_key), path)
|
||||
except ObjectMissing:
|
||||
return None
|
||||
return json.loads(path.read_text())
|
||||
|
||||
|
||||
def local_counts(manifest_path: str | None) -> dict | None:
|
||||
"""The counts in a dump's manifest file on this server, or None."""
|
||||
try:
|
||||
return json.loads(Path(manifest_path).read_text()).get("counts")
|
||||
except (TypeError, OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def scratch_name(db_url: str) -> str:
|
||||
"""The temporary database's name: the live one's, plus SCRATCH_SUFFIX."""
|
||||
return f"{make_url(db_url).database}{SCRATCH_SUFFIX}"
|
||||
|
||||
|
||||
def scratch_url(db_url: str) -> str:
|
||||
return make_url(db_url).set(database=scratch_name(db_url)).render_as_string(
|
||||
hide_password=False)
|
||||
|
||||
|
||||
def _on_server(db_url: str, statement) -> None:
|
||||
"""Run a CREATE or DROP DATABASE, which can't run inside a transaction."""
|
||||
import psycopg
|
||||
|
||||
with psycopg.connect(backup_service._libpq_url(db_url), autocommit=True) as conn:
|
||||
conn.execute(statement)
|
||||
|
||||
|
||||
def drop_scratch(db_url: str) -> None:
|
||||
"""Drop the temporary database if it is there, ending any connection to it."""
|
||||
from psycopg import sql
|
||||
|
||||
_on_server(db_url, sql.SQL("DROP DATABASE IF EXISTS {} WITH (FORCE)").format(
|
||||
sql.Identifier(scratch_name(db_url))))
|
||||
|
||||
|
||||
def _create_scratch(db_url: str) -> None:
|
||||
from psycopg import sql
|
||||
|
||||
_on_server(db_url, sql.SQL("CREATE DATABASE {} TEMPLATE template0").format(
|
||||
sql.Identifier(scratch_name(db_url))))
|
||||
|
||||
|
||||
def trial_restore(*, db_url: str, dump_path: Path, recorded: dict | None,
|
||||
note: Callable[[str], None] = lambda phase: None) -> dict:
|
||||
"""Load the dump at `dump_path` into the temporary database, compare it,
|
||||
bring it up to this version, and drop it. `note` is told each phase as it
|
||||
starts: "loading", "comparing", "upgrading". Raises RestoreFailed (or
|
||||
S3Error from the upgrade) when the dump doesn't load or can't be brought
|
||||
up; the temporary database is dropped either way."""
|
||||
scratch = scratch_url(db_url)
|
||||
if make_url(scratch).database == make_url(db_url).database:
|
||||
raise ValueError("The test database would be the live one.")
|
||||
drop_scratch(db_url) # one left by a test cut off by a restart
|
||||
_create_scratch(db_url)
|
||||
try:
|
||||
note("loading")
|
||||
backup_service.restore_db(db_url=scratch, sql_path=dump_path)
|
||||
note("comparing")
|
||||
restored = backup_service.read_counts(scratch)
|
||||
live = backup_service.read_counts(db_url)
|
||||
note("upgrading")
|
||||
backup_restore.migrate(db_name=scratch_name(db_url))
|
||||
upgraded = backup_service.read_counts(scratch)["alembic"]
|
||||
finally:
|
||||
drop_scratch(db_url)
|
||||
return {**compare(recorded, restored, live),
|
||||
"alembic_dump": restored["alembic"], "alembic_now": upgraded}
|
||||
|
||||
|
||||
def compare(recorded: dict | None, restored: dict, live: dict) -> dict:
|
||||
"""Each table's rows in the dump (as recorded), in the restored copy, and
|
||||
in the live database. With counts recorded, every table must come back
|
||||
with exactly its count; `mismatched` names those that didn't."""
|
||||
rec = (recorded or {}).get("tables")
|
||||
names = sorted(set(restored["tables"]) | set(rec or {}) | set(live["tables"]))
|
||||
tables = [
|
||||
{"table": n, "dump": rec.get(n) if rec is not None else None,
|
||||
"restored": restored["tables"].get(n), "live": live["tables"].get(n)}
|
||||
for n in names
|
||||
]
|
||||
mismatched = [t["table"] for t in tables
|
||||
if rec is not None and t["table"] in rec and t["restored"] != t["dump"]]
|
||||
return {
|
||||
"counted": rec is not None,
|
||||
"tables": tables,
|
||||
"mismatched": mismatched,
|
||||
"rows_restored": sum(v for v in restored["tables"].values()),
|
||||
}
|
||||
|
||||
|
||||
def summary(result: dict) -> str:
|
||||
"""One sentence on how a test went."""
|
||||
n = len([t for t in result["tables"] if t["restored"] is not None])
|
||||
rows = f"{result['rows_restored']:,}"
|
||||
if result["mismatched"]:
|
||||
shown = ", ".join(
|
||||
f"{t['table']} ({t['dump']:,} in the dump, "
|
||||
+ ("table missing)" if t["restored"] is None else f"{t['restored']:,} restored)")
|
||||
for t in result["tables"] if t["table"] in result["mismatched"][:5]
|
||||
)
|
||||
more = len(result["mismatched"]) - 5
|
||||
return (f"{len(result['mismatched'])} table(s) came back different: {shown}"
|
||||
+ (f", and {more} more" if more > 0 else "") + ".")
|
||||
if result["counted"]:
|
||||
return f"All {n} tables came back with the {rows} rows the dump recorded."
|
||||
return (f"The dump loaded cleanly: {n} tables, {rows} rows. It was taken before "
|
||||
"dumps recorded their counts, so it is shown beside the live database instead.")
|
||||
@@ -1044,45 +1044,75 @@ def cancel_audit_run(session: Session, *, audit_id: int) -> None:
|
||||
_ARCHIVE_EXT_FOR_FORMAT = {"zip": ".zip", "rar": ".rar", "7z": ".7z"}
|
||||
|
||||
|
||||
def _stage_archive(archive_path: Path, staged: Path) -> None:
|
||||
"""Put the stored archive at `staged` for the importer to read: a symlink,
|
||||
so a multi-gigabyte archive isn't copied, or a copy where links aren't
|
||||
possible."""
|
||||
import shutil
|
||||
|
||||
staged.unlink(missing_ok=True)
|
||||
try:
|
||||
staged.symlink_to(archive_path)
|
||||
except OSError:
|
||||
shutil.copy2(archive_path, staged)
|
||||
|
||||
|
||||
def _archive_folder_name(original_filename: str | None, attachment_id: int) -> str:
|
||||
"""The library folder a post-less archive's members land in: the archive's
|
||||
own name without its extension, made safe for a path."""
|
||||
stem = Path(original_filename or "").stem.strip().strip(".")
|
||||
stem = "".join(c if c not in '/\\:*?"<>|' else "_" for c in stem)[:120].strip()
|
||||
return stem or f"archive-{attachment_id}"
|
||||
|
||||
|
||||
def _reextract_archive_to_post(
|
||||
importer, archive_path: Path, post, source_row, artist, images_root: Path,
|
||||
*, folder: str = "",
|
||||
) -> list[int]:
|
||||
"""Extract one stored archive and link its members to `post`.
|
||||
"""Extract one stored archive and import its members.
|
||||
|
||||
The stored attachment has no adjacent sidecar (it lives in the sha-addressed
|
||||
attachment store). Stage a copy + a reconstructed sidecar UNDER the artist's
|
||||
library dir (`images_root/<slug>/<platform>/<post>/`) — the importer
|
||||
re-derives the artist from the path AND copies members relative to it, so the
|
||||
members land in the real library and resolve to the right artist — then re-run
|
||||
`attach_in_place`: the archive extracts and `find_or_create_post` re-attaches
|
||||
the members to the SAME Post (source_id + external_post_id). Removes only the
|
||||
staged archive + sidecar afterward; the imported member files stay. Returns
|
||||
the new member image ids.
|
||||
attachment store), so the archive is staged under the artist's library dir
|
||||
and `attach_in_place` re-run on it: the importer derives the artist from the
|
||||
path and copies members relative to it, so they land in the real library.
|
||||
|
||||
- With a `post`, the stage is `images_root/<slug>/<platform>/<post>/` beside
|
||||
a reconstructed sidecar, so `find_or_create_post` re-attaches the members
|
||||
to that same Post (source_id + external_post_id).
|
||||
- Without one (a filesystem import that had no sidecar), the stage is
|
||||
`images_root/<slug>/<folder>/` with no sidecar, and the members belong to
|
||||
the artist alone, as a filesystem import of the archive would have left
|
||||
them.
|
||||
|
||||
Only the staged archive and sidecar are removed afterward; the imported
|
||||
member files stay. Returns the new member image ids.
|
||||
"""
|
||||
import json
|
||||
import shutil
|
||||
|
||||
from .archive_extractor import detect_archive_format
|
||||
|
||||
fmt = detect_archive_format(archive_path)
|
||||
ext = _ARCHIVE_EXT_FOR_FORMAT.get(fmt or "", ".zip")
|
||||
platform = source_row.platform if source_row is not None else "imported"
|
||||
sidecar = {
|
||||
"category": source_row.platform if source_row is not None
|
||||
else (post.raw_metadata or {}).get("category"),
|
||||
"id": post.external_post_id,
|
||||
"title": post.post_title or "",
|
||||
"content": post.description or "",
|
||||
"published_at": post.post_date.isoformat() if post.post_date else None,
|
||||
"url": post.post_url,
|
||||
}
|
||||
work = images_root / artist.slug / platform / str(post.external_post_id)
|
||||
if post is not None:
|
||||
platform = source_row.platform if source_row is not None else "imported"
|
||||
work = images_root / artist.slug / platform / str(post.external_post_id)
|
||||
else:
|
||||
work = images_root / artist.slug / folder
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
staged = work / f"archive{ext}" # clean ext → is_archive + find_sidecar
|
||||
sidecar_path = staged.with_suffix(".json")
|
||||
try:
|
||||
shutil.copy2(archive_path, staged)
|
||||
sidecar_path.write_text(json.dumps(sidecar))
|
||||
_stage_archive(archive_path, staged)
|
||||
if post is not None:
|
||||
sidecar_path.write_text(json.dumps({
|
||||
"category": source_row.platform if source_row is not None
|
||||
else (post.raw_metadata or {}).get("category"),
|
||||
"id": post.external_post_id,
|
||||
"title": post.post_title or "",
|
||||
"content": post.description or "",
|
||||
"published_at": post.post_date.isoformat() if post.post_date else None,
|
||||
"url": post.post_url,
|
||||
}))
|
||||
res = importer.attach_in_place(staged, artist=artist, source=source_row)
|
||||
return list(res.member_image_ids or [])
|
||||
finally:
|
||||
@@ -1124,8 +1154,9 @@ def reextract_archive_attachments(
|
||||
|
||||
summary = {
|
||||
"scanned": 0, "archives": 0, "members_imported": 0,
|
||||
"posts_touched": 0, "skipped_no_post": 0, "skipped_no_artist": 0,
|
||||
"errors": 0, "partial": False, "resume_after_id": after_id,
|
||||
"posts_touched": 0, "without_post": 0, "already_extracted": 0,
|
||||
"skipped_no_artist": 0, "errors": 0, "partial": False,
|
||||
"resume_after_id": after_id,
|
||||
}
|
||||
settings = ImportSettings.load_sync(session)
|
||||
importer = Importer(
|
||||
@@ -1150,15 +1181,17 @@ def reextract_archive_attachments(
|
||||
except OSError:
|
||||
continue
|
||||
summary["archives"] += 1
|
||||
if att.post_id is None:
|
||||
summary["skipped_no_post"] += 1
|
||||
continue
|
||||
post = session.get(Post, att.post_id)
|
||||
if post is None:
|
||||
summary["skipped_no_post"] += 1
|
||||
# Members already recorded as coming from this archive: done before.
|
||||
if session.execute(
|
||||
select(ImageProvenance.id)
|
||||
.where(ImageProvenance.from_attachment_id == att.id)
|
||||
.limit(1)
|
||||
).first() is not None:
|
||||
summary["already_extracted"] += 1
|
||||
continue
|
||||
post = session.get(Post, att.post_id) if att.post_id else None
|
||||
artist = session.get(Artist, att.artist_id) if att.artist_id else None
|
||||
if artist is None and post.artist_id:
|
||||
if artist is None and post is not None and post.artist_id:
|
||||
artist = session.get(Artist, post.artist_id)
|
||||
if artist is None or not artist.slug:
|
||||
# The importer re-derives the artist from the staged path, so we need
|
||||
@@ -1166,20 +1199,35 @@ def reextract_archive_attachments(
|
||||
# subscription posts; skip rather than orphan the members.)
|
||||
summary["skipped_no_artist"] += 1
|
||||
continue
|
||||
source_row = session.get(Source, post.source_id) if post.source_id else None
|
||||
source_row = (
|
||||
session.get(Source, post.source_id)
|
||||
if post is not None and post.source_id else None
|
||||
)
|
||||
if post is None:
|
||||
summary["without_post"] += 1
|
||||
# Logged on the way in as well as out, so an archive that stalls a
|
||||
# chunk is named in the log even if the worker dies holding it.
|
||||
size_mb = (att.size_bytes or 0) / 1e6
|
||||
log.info("re-extract: attachment %s (%.0f MB) starting", att.id, size_mb)
|
||||
archive_start = time.monotonic()
|
||||
try:
|
||||
ids = _reextract_archive_to_post(
|
||||
importer, stored, post, source_row, artist, images_root,
|
||||
folder=_archive_folder_name(att.original_filename, att.id),
|
||||
)
|
||||
session.commit()
|
||||
except Exception as exc: # one bad archive must not strand the rest
|
||||
session.rollback()
|
||||
summary["errors"] += 1
|
||||
log.warning("re-extract failed for attachment %s: %s", att.id, exc)
|
||||
log.warning("re-extract failed for attachment %s after %.1fs: %s",
|
||||
att.id, time.monotonic() - archive_start, exc)
|
||||
continue
|
||||
log.info("re-extract: attachment %s done in %.1fs, %d new image(s)",
|
||||
att.id, time.monotonic() - archive_start, len(ids))
|
||||
if ids:
|
||||
summary["members_imported"] += len(ids)
|
||||
summary["posts_touched"] += 1
|
||||
if post is not None:
|
||||
summary["posts_touched"] += 1
|
||||
enqueue_ids.extend(ids)
|
||||
|
||||
# Time-box the chunk. resume_after_id already points at this attachment,
|
||||
|
||||
@@ -21,7 +21,7 @@ from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..models import Source
|
||||
from .gallery_dl import ErrorType
|
||||
from .download_result import ErrorType
|
||||
|
||||
|
||||
async def get_or_create[T](
|
||||
|
||||
@@ -32,6 +32,13 @@ Two deliberate departures, both about the walk order, neither about content:
|
||||
the walk. gallery-dl skips only nested channels; one private thread the
|
||||
token cannot read would otherwise stop every channel after it.
|
||||
|
||||
And one about content: a message is taken only if it has an image or video
|
||||
ATTACHMENT (`has_visual_attachment`). Discord is where creators chat as well
|
||||
as post, and the operator wants the art, not the conversation (2026-09-28): a
|
||||
text line, a lone archive or PSD, a pasted link's preview or a Tenor GIF is
|
||||
chat. A message that passes keeps every file gallery-dl would take, numbered
|
||||
as gallery-dl numbers them, so the on-disk names still match.
|
||||
|
||||
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
|
||||
@@ -42,13 +49,13 @@ import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from urllib.parse import unquote
|
||||
|
||||
import requests
|
||||
|
||||
from .native_ingest_common import (
|
||||
MediaItem,
|
||||
NativeAuthError,
|
||||
NativeDriftError,
|
||||
NativeIngestError,
|
||||
@@ -68,6 +75,9 @@ _THREADS_BATCH = 25
|
||||
# it is honoured when present; 60s is the fallback and the cap.
|
||||
_MAX_429_RETRIES = 4
|
||||
_429_WAIT_SECONDS = 60.0
|
||||
# How far back the poster picker looks: two pages. Enough to see who posts in
|
||||
# a channel, few enough to answer while the operator waits (#4491).
|
||||
_POSTER_SCAN = 200
|
||||
|
||||
# https://discord.com/developers/docs/resources/message#message-object-message-types
|
||||
# DEFAULT, REPLY, CHAT_INPUT_COMMAND — the ones that carry user content.
|
||||
@@ -79,6 +89,13 @@ _FORUM = frozenset({15, 16}) # forum, media: threads only
|
||||
_CATEGORY = 4
|
||||
_SERVER_WALK = _TEXT | _FORUM
|
||||
_EMBED_TYPES = frozenset({"image", "gifv", "video"})
|
||||
# What makes a message art rather than chat: an attached file FC imports as an
|
||||
# image or video. Mirrors importer.ALL_EXTS (not imported: that module is the
|
||||
# import pipeline, and this one is a network client).
|
||||
_VISUAL_EXTS = frozenset({
|
||||
"png", "jpg", "jpeg", "gif", "webp", "bmp", "tif", "tiff",
|
||||
"mp4", "mov", "avi", "mkv", "webm", "m4v", "wmv", "flv",
|
||||
})
|
||||
|
||||
_URL_RE = re.compile(
|
||||
r"^(?:https?://)?(?:www\.|ptb\.|canary\.)?discord(?:app)?\.com/channels/"
|
||||
@@ -163,27 +180,80 @@ def message_text(message: dict) -> str:
|
||||
return "\n".join(p for p in parts if p)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MediaItem:
|
||||
"""One file of a Discord message. `filename`/`extension` are gallery-dl's
|
||||
split of the URL; `num` is its 1-based position across the message's files,
|
||||
which is what names it on disk.
|
||||
def _message_and_snapshots(message: dict) -> list[dict]:
|
||||
"""The message itself, then each forwarded message it carries."""
|
||||
return [message] + [
|
||||
(s or {}).get("message") or {}
|
||||
for s in message.get("message_snapshots") or []
|
||||
if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES
|
||||
]
|
||||
|
||||
`media_id` is what the seen-ledger keys on, and it is deliberately NOT
|
||||
`num`: an edit that removes a file renumbers the ones after it, and a
|
||||
positional key would then call a different file seen. It is the
|
||||
attachment's id, or for an embed (which has none) a hash of its URL path —
|
||||
the query string is a signature that changes on every fetch. `filehash` is
|
||||
always None; nothing in a signed CDN URL is a content hash."""
|
||||
|
||||
url: str
|
||||
filename: str
|
||||
extension: str
|
||||
kind: str
|
||||
post_id: str
|
||||
num: int
|
||||
media_id: str
|
||||
filehash: str | None = None
|
||||
def _is_visual(attachment: dict) -> bool:
|
||||
ctype = (attachment.get("content_type") or "").lower()
|
||||
if ctype.startswith(("image/", "video/")):
|
||||
return True
|
||||
name = attachment.get("filename") or attachment.get("url") or ""
|
||||
return nameext_from_url(name)[1] in _VISUAL_EXTS
|
||||
|
||||
|
||||
def has_visual_attachment(message: dict) -> bool:
|
||||
"""Does this message, or a message it forwards, have an image or video
|
||||
attached? The one test for "art, not chat" — see the module docstring."""
|
||||
return any(
|
||||
att.get("url") and _is_visual(att)
|
||||
for snap in _message_and_snapshots(message)
|
||||
for att in snap.get("attachments") or []
|
||||
)
|
||||
|
||||
|
||||
_SQUASH = re.compile(r"[\W_]+", re.UNICODE)
|
||||
# Words a server name wraps around its creator's: "Todding's Server",
|
||||
# "The Official Todding Discord".
|
||||
_SERVER_WORDS = frozenset({"server", "discord", "the", "official", "community", "s"})
|
||||
|
||||
|
||||
def _squash(text: str | None) -> str:
|
||||
return _SQUASH.sub("", (text or "").lower())
|
||||
|
||||
|
||||
def _server_tokens(server_name: str | None) -> list[str]:
|
||||
name = (server_name or "").lower().replace("'s", " ").replace("\u2019s", " ")
|
||||
words = [w for w in _SQUASH.split(name) if w and w not in _SERVER_WORDS]
|
||||
tokens = [w for w in words if len(w) >= 3]
|
||||
joined = "".join(words)
|
||||
if len(joined) >= 3 and joined not in tokens:
|
||||
tokens.append(joined)
|
||||
return tokens
|
||||
|
||||
|
||||
def rank_posters(server_name: str | None, owner_id, posters: list[dict]) -> list[dict]:
|
||||
"""Mark who is probably the creator a server is named for, and order the
|
||||
list for picking: suggested first, then by images posted, then messages.
|
||||
|
||||
Two signals, each a reason the picker shows beside the name:
|
||||
- they own the server;
|
||||
- their username or display name matches the server's name, once
|
||||
"'s Server", "Official", "Discord" and punctuation are taken off
|
||||
("Todding's Server" and "todding").
|
||||
Neither signal alone is proof, which is why this suggests and never
|
||||
decides: the operator ticks the list. Bots are never suggested."""
|
||||
tokens = _server_tokens(server_name)
|
||||
owner = str(owner_id) if owner_id else None
|
||||
ranked = []
|
||||
for p in posters:
|
||||
reasons = []
|
||||
if owner and p.get("id") == owner:
|
||||
reasons.append("owns the server")
|
||||
names = [n for n in (_squash(p.get("username")), _squash(p.get("global_name"))) if len(n) >= 3]
|
||||
if any(t in n or n in t for t in tokens for n in names):
|
||||
reasons.append("name matches the server")
|
||||
ranked.append({**p, "reasons": reasons, "suggested": False})
|
||||
best = max((len(r["reasons"]) for r in ranked if not r.get("bot")), default=0)
|
||||
for r in ranked:
|
||||
r["suggested"] = best > 0 and not r.get("bot") and len(r["reasons"]) == best
|
||||
ranked.sort(key=lambda r: (not r["suggested"], -r.get("images", 0), -r.get("messages", 0)))
|
||||
return ranked
|
||||
|
||||
|
||||
class DiscordClient:
|
||||
@@ -212,6 +282,8 @@ class DiscordClient:
|
||||
self._server: dict = {}
|
||||
self._channels: dict[str, dict] = {}
|
||||
self._skip_feed = False
|
||||
# Whose messages the walk takes (`only_from`); empty takes everyone's.
|
||||
self._authors: frozenset[str] = frozenset()
|
||||
|
||||
# -- request -----------------------------------------------------------
|
||||
|
||||
@@ -373,6 +445,43 @@ class DiscordClient:
|
||||
if meta["channel_type"] in _SERVER_WALK:
|
||||
yield from self._feeds(meta["channel_id"], safe=True)
|
||||
|
||||
def only_from(self, authors) -> None:
|
||||
"""Take only messages posted by these people: user ids, usernames or
|
||||
display names, any case. A creator's server is full of other members
|
||||
posting their own pictures; the source is subscribed to the creator.
|
||||
Empty or None takes everyone's, as before."""
|
||||
self._authors = frozenset(
|
||||
str(a).strip().lower() for a in authors or () if str(a).strip()
|
||||
)
|
||||
|
||||
def _author_wanted(self, message: dict) -> bool:
|
||||
if not self._authors:
|
||||
return True
|
||||
author = message.get("author") or {}
|
||||
names = (author.get("id"), author.get("username"), author.get("global_name"))
|
||||
return any(str(n).lower() in self._authors for n in names if n)
|
||||
|
||||
def source_label(self, url: str) -> str | None:
|
||||
"""What the source is called in Discord — `Server · #channel`, a
|
||||
thread as `#parent › thread`, a whole server by its name — from the
|
||||
metadata the walk loaded. None when the walk never got that far."""
|
||||
try:
|
||||
_, channel_id = parse_source_url(url)
|
||||
except DiscordAPIError:
|
||||
return None
|
||||
server = self._server.get("server") or None
|
||||
if channel_id is None:
|
||||
return server
|
||||
meta = self._channels.get(channel_id) or {}
|
||||
name = meta.get("channel")
|
||||
if not name:
|
||||
return None
|
||||
if name != "DMs":
|
||||
name = f"#{name}"
|
||||
if meta.get("is_thread") and meta.get("parent"):
|
||||
name = f"#{meta['parent']} › {meta['channel']}"
|
||||
return f"{server} · {name}" if server else name
|
||||
|
||||
def skip_feed(self) -> None:
|
||||
"""Optional core seam (#4413): end the current channel and go on to the
|
||||
next one. A tick's early-out means THIS channel has nothing new, not
|
||||
@@ -440,6 +549,8 @@ class DiscordClient:
|
||||
for message in messages:
|
||||
if message.get("type") not in MESSAGE_TYPES:
|
||||
continue
|
||||
if not self._author_wanted(message):
|
||||
continue
|
||||
message["_meta"] = meta
|
||||
yield message, meta, page_cursor
|
||||
if self._skip_feed:
|
||||
@@ -454,13 +565,14 @@ class DiscordClient:
|
||||
def extract_media(post: dict, included: dict | None = None) -> list[MediaItem]:
|
||||
"""gallery-dl's file list for one message: attachments, then the first
|
||||
of video/image/thumbnail `proxy_url` of each file-bearing embed, then
|
||||
the same for every forwarded snapshot; numbered from 1 across them."""
|
||||
the same for every forwarded snapshot; numbered from 1 across them.
|
||||
|
||||
Empty for a message with no image or video attached: that is chat,
|
||||
whatever else it carries (`has_visual_attachment`)."""
|
||||
if not has_visual_attachment(post):
|
||||
return []
|
||||
mid = str(post.get("id") or "")
|
||||
snapshots = [post] + [
|
||||
(s or {}).get("message") or {}
|
||||
for s in post.get("message_snapshots") or []
|
||||
if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES
|
||||
]
|
||||
snapshots = _message_and_snapshots(post)
|
||||
found: list[tuple[str, str, str | None]] = []
|
||||
for snap in snapshots:
|
||||
for att in snap.get("attachments") or []:
|
||||
@@ -498,10 +610,10 @@ class DiscordClient:
|
||||
"""`(message:<id>, <id>)` — gates the message record through the seen
|
||||
ledger, like `post:<id>` on the other platforms.
|
||||
|
||||
None for a message with no files. gallery-dl wrote a sidecar only
|
||||
beside a file, so a text-only chat line never became a post, and the
|
||||
drop grouping (discord_grouping) is built on that: a channel's chatter
|
||||
recorded as posts would bury the drops it exists to surface."""
|
||||
None for a message `extract_media` takes nothing from, i.e. one with
|
||||
no image or video attached. The drop grouping (discord_grouping) is
|
||||
built on that: a channel's chatter recorded as posts would bury the
|
||||
drops it exists to surface."""
|
||||
mid = post.get("id")
|
||||
mid = str(mid) if mid is not None else ""
|
||||
if not mid or not cls.extract_media(post):
|
||||
@@ -528,6 +640,61 @@ class DiscordClient:
|
||||
pass
|
||||
return out
|
||||
|
||||
def recent_posters(
|
||||
self, server_id: str | None, channel_id: str, *, max_messages: int = _POSTER_SCAN,
|
||||
) -> dict:
|
||||
"""Who has posted lately in a channel, for picking a source's poster
|
||||
list (#4491): the newest `max_messages` messages, tallied per author.
|
||||
Deliberately shallow — it answers "who posts here", not "who ever did".
|
||||
Every poster comes back with their stable id, which is what the list
|
||||
stores: a username can change on a whim, an id can't."""
|
||||
server: dict = {}
|
||||
if server_id:
|
||||
try:
|
||||
server = self._get(f"/guilds/{server_id}") or {}
|
||||
except DiscordAPIError:
|
||||
server = {}
|
||||
tally: dict[str, dict] = {}
|
||||
before = None
|
||||
seen = 0
|
||||
while seen < max_messages:
|
||||
page = self._get(
|
||||
f"/channels/{channel_id}/messages",
|
||||
{"limit": min(_MESSAGES_BATCH, max_messages - seen), "before": before},
|
||||
)
|
||||
if not isinstance(page, list) or not page:
|
||||
break
|
||||
for message in page:
|
||||
seen += 1
|
||||
if message.get("type") not in MESSAGE_TYPES:
|
||||
continue
|
||||
author = message.get("author") or {}
|
||||
aid = str(author.get("id") or "")
|
||||
if not aid:
|
||||
continue
|
||||
row = tally.setdefault(aid, {
|
||||
"id": aid,
|
||||
"username": author.get("username"),
|
||||
"global_name": author.get("global_name"),
|
||||
"bot": bool(author.get("bot")),
|
||||
"messages": 0,
|
||||
"images": 0,
|
||||
})
|
||||
row["messages"] += 1
|
||||
if has_visual_attachment(message):
|
||||
row["images"] += 1
|
||||
if len(page) < _MESSAGES_BATCH:
|
||||
break
|
||||
before = str(page[-1]["id"])
|
||||
return {
|
||||
"server": server.get("name") or None,
|
||||
"owner_id": str(server["owner_id"]) if server.get("owner_id") else None,
|
||||
"scanned": seen,
|
||||
"posters": rank_posters(
|
||||
server.get("name"), server.get("owner_id"), list(tally.values()),
|
||||
),
|
||||
}
|
||||
|
||||
def verify_auth(self, url: str) -> tuple[bool | None, str]:
|
||||
"""Is the token valid, and can it see what the source names?"""
|
||||
try:
|
||||
|
||||
@@ -30,10 +30,7 @@ PURE: no DB; the seen-skip is an injected predicate.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
@@ -42,13 +39,11 @@ import requests
|
||||
from .discord_client import firefox_user_agent, message_text
|
||||
from .native_ingest_common import (
|
||||
BaseNativeDownloader,
|
||||
MediaOutcome,
|
||||
MediaItem,
|
||||
PostRecordOutcome,
|
||||
make_session,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
PLATFORM = "discord"
|
||||
_CONTROL = re.compile("[\x00-\x1f\x7f]")
|
||||
# gallery-dl falls back to the response's type for a URL with no extension;
|
||||
@@ -111,67 +106,19 @@ class DiscordDownloader(BaseNativeDownloader):
|
||||
}),
|
||||
)
|
||||
|
||||
def download_post(
|
||||
self,
|
||||
post: dict,
|
||||
media_items: list,
|
||||
artist_slug: str,
|
||||
*,
|
||||
is_seen: Callable[[object], bool] = lambda m: False,
|
||||
should_stop: Callable[[], bool] = lambda: False,
|
||||
recapture: bool = False,
|
||||
) -> list[MediaOutcome]:
|
||||
"""Every file of one message; per-file outcomes, one failure isolated."""
|
||||
folder = channel_dir(self.images_root, artist_slug, post)
|
||||
outcomes: list[MediaOutcome] = []
|
||||
for media in media_items:
|
||||
if should_stop():
|
||||
break
|
||||
try:
|
||||
outcomes.append(self._download_one(
|
||||
post, media, folder, artist_slug, is_seen, recapture=recapture,
|
||||
))
|
||||
except Exception as exc: # resilient: isolate one item's failure
|
||||
log.warning(
|
||||
"Discord media failed (message %s, file %d): %s",
|
||||
post.get("id"), media.num, exc,
|
||||
)
|
||||
outcomes.append(
|
||||
MediaOutcome(media=media, status="error", path=None, error=str(exc))
|
||||
)
|
||||
return outcomes
|
||||
# The download loop is BaseNativeDownloader's; these seams give it
|
||||
# gallery-dl's Discord layout.
|
||||
|
||||
def _download_one(
|
||||
self,
|
||||
post: dict,
|
||||
media,
|
||||
folder: Path,
|
||||
artist_slug: str,
|
||||
is_seen: Callable[[object], bool],
|
||||
*,
|
||||
recapture: bool = False,
|
||||
) -> MediaOutcome:
|
||||
seen = is_seen(media)
|
||||
if seen and not recapture:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
stem = media_stem(post, media)
|
||||
path = folder / f"{stem}.{media.extension or _NO_EXTENSION}"
|
||||
if path.exists(): # tier-2: gallery-dl (or an earlier walk) wrote it
|
||||
return MediaOutcome(media=media, status="skipped_disk", path=path, error=None)
|
||||
if seen: # recapture never re-fetches a seen file that is gone
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
def _post_folder(self, post: dict, artist_slug: str) -> Path:
|
||||
return channel_dir(self.images_root, artist_slug, post)
|
||||
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
if self._rate_limit > 0:
|
||||
time.sleep(self._rate_limit)
|
||||
out = self._fetch_get(media.url, path)
|
||||
reason, quarantined = self._validate_path(out, artist_slug, media.url)
|
||||
if reason is not None:
|
||||
return MediaOutcome(media=media, status="quarantined", path=quarantined, error=reason)
|
||||
sidecar = {"category": PLATFORM, "message_id": str(post.get("id") or "")}
|
||||
sidecar["source_url"] = media.url
|
||||
(folder / f"{stem}.json").write_text(json.dumps(sidecar, indent=2))
|
||||
return MediaOutcome(media=media, status="downloaded", path=out, error=None)
|
||||
def _media_name(self, post: dict, media: MediaItem, index: int) -> str:
|
||||
return f"{media_stem(post, media)}.{media.extension or _NO_EXTENSION}"
|
||||
|
||||
def _media_sidecar(self, post: dict, media: MediaItem) -> dict:
|
||||
# `message_id`, never `id`: `id` outranks it in the post-id chain.
|
||||
return {"category": PLATFORM, "message_id": str(post.get("id") or ""),
|
||||
"source_url": media.url}
|
||||
|
||||
def write_post_record(
|
||||
self, post: dict, artist_slug: str, *, revisit: bool = False,
|
||||
@@ -195,6 +142,7 @@ class DiscordDownloader(BaseNativeDownloader):
|
||||
"parent": meta.get("parent"),
|
||||
"is_thread": meta.get("is_thread"),
|
||||
"author": author.get("username"),
|
||||
"author_name": author.get("global_name"),
|
||||
"author_id": author.get("id"),
|
||||
"message": body,
|
||||
"date": post.get("timestamp"),
|
||||
|
||||
@@ -13,6 +13,11 @@ Two things differ from the cookie platforms:
|
||||
something; a Discord drop is routinely files and nothing else, so on
|
||||
Discord that is an ordinary backfill, not a broken parser.
|
||||
|
||||
And two things only a Discord source has: `discord_authors` in its
|
||||
config_overrides limits it to the creator's own messages (a server's other
|
||||
members post pictures too), and it learns its own name — the server and
|
||||
channel it walks — into `source.display_name`.
|
||||
|
||||
`campaign_id` is the source URL (a server, channel, thread or category link).
|
||||
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
@@ -20,20 +25,23 @@ FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
from ..models import DiscordFailedMedia, DiscordSeenMedia
|
||||
from .discord_client import DiscordAPIError, DiscordClient, MediaItem
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from ..models import DiscordFailedMedia, DiscordSeenMedia, Source
|
||||
from .discord_client import DiscordAPIError, DiscordClient
|
||||
from .discord_downloader import DiscordDownloader
|
||||
from .ingest_core import Ingester
|
||||
from .native_ingest_common import ledger_key
|
||||
|
||||
_LEDGER_KEY_MAX = 128
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _ledger_key(media: MediaItem) -> str:
|
||||
"""`<message_id>:<media_id>` — stable across edits (see MediaItem)."""
|
||||
return f"{media.post_id}:{media.media_id}"[:_LEDGER_KEY_MAX]
|
||||
# `config_overrides` key: the people whose messages this source takes (user
|
||||
# ids, usernames or display names). Absent or empty takes everyone's.
|
||||
AUTHORS_KEY = "discord_authors"
|
||||
|
||||
|
||||
class DiscordIngester(Ingester):
|
||||
@@ -67,13 +75,53 @@ class DiscordIngester(Ingester):
|
||||
failed_model=DiscordFailedMedia,
|
||||
seen_constraint="uq_discord_seen_media_source_id",
|
||||
failed_constraint="uq_discord_failed_media_source_id",
|
||||
ledger_key=_ledger_key,
|
||||
ledger_key=ledger_key,
|
||||
platform="discord",
|
||||
error_base=DiscordAPIError,
|
||||
drift_label="Discord API",
|
||||
body_canary=False,
|
||||
)
|
||||
|
||||
def run(self, **kwargs):
|
||||
"""The core walk, bracketed by the two things only a Discord source
|
||||
has: whose messages it takes, read before the walk, and the name of
|
||||
what it walks, written after it (the walk is what loads that name)."""
|
||||
source_id = kwargs["source_id"]
|
||||
self.client.only_from(self._source_authors(source_id))
|
||||
try:
|
||||
return super().run(**kwargs)
|
||||
finally:
|
||||
self._record_display_name(source_id, kwargs["campaign_id"])
|
||||
|
||||
def _source_authors(self, source_id: int) -> list:
|
||||
if self.session_factory is None:
|
||||
return []
|
||||
with self.session_factory() as session:
|
||||
overrides = session.execute(
|
||||
select(Source.config_overrides).where(Source.id == source_id)
|
||||
).scalar_one_or_none() or {}
|
||||
authors = overrides.get(AUTHORS_KEY) or []
|
||||
return authors if isinstance(authors, list) else []
|
||||
|
||||
def _record_display_name(self, source_id: int, url: str) -> None:
|
||||
"""Refreshed on every walk, never written once: a renamed channel
|
||||
should read as its new name. A name the walk couldn't read leaves the
|
||||
stored one alone, and a failure here never fails the walk."""
|
||||
label = self.client.source_label(url)
|
||||
if not label or self.session_factory is None:
|
||||
return
|
||||
try:
|
||||
with self.session_factory() as session:
|
||||
session.execute(
|
||||
update(Source)
|
||||
.where(Source.id == source_id)
|
||||
.where(Source.display_name.is_distinct_from(label))
|
||||
.values(display_name=label)
|
||||
)
|
||||
session.commit()
|
||||
except Exception as exc: # a name is decoration — never fail the walk
|
||||
log.warning("Discord: couldn't record source %s's name: %s", source_id, exc)
|
||||
|
||||
|
||||
async def verify_discord_credential(url: str, auth_token: str | None) -> tuple[bool | None, str]:
|
||||
"""The uniform `(ok, message)` probe: is the token valid, and can its
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Remove a Discord source's posts by people outside its poster list (#4486).
|
||||
|
||||
A creator's server is full of other members posting their own pictures. The
|
||||
poster list (`discord_authors`, #4481) stops a walk taking them; this removes
|
||||
the ones taken before the list existed. The operator sets the list first, then
|
||||
previews what would go, then applies.
|
||||
|
||||
## What goes
|
||||
|
||||
A post of this source whose record names its poster (`author_id` / `author`,
|
||||
written by `DiscordDownloader.write_post_record`) and names someone NOT on the
|
||||
list. Two kinds of post are never touched:
|
||||
|
||||
* a post whose record names no poster — a message recorded before the native
|
||||
ingester, whose poster FC cannot tell. Counted as `unknown`, never guessed;
|
||||
* a synthetic drop post (`synthesized_by`) — FC authored it, and it is handled
|
||||
below as a consequence, not matched as a poster's post.
|
||||
|
||||
An image goes when every REAL post it belongs to is going. A link to a
|
||||
synthetic drop does not keep an image: the drop only references its members'
|
||||
images. An image also on a kept post — the creator re-posting a piece someone
|
||||
else shared — stays.
|
||||
|
||||
A drop that absorbed a removed message is deleted outright. Deleting a drop is
|
||||
its undo (discord_grouping's honesty rule): its remaining members return to
|
||||
the feed and the grouping sweep regroups them on its next run, so no half-
|
||||
rebuilt drop keeps text or thumbnails from someone it no longer contains.
|
||||
|
||||
Preview and apply spread the same predicates (rule 93, snippet #3087).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import and_, delete, exists, func, or_, select
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from ..models import ImageProvenance, ImageRecord, Post, PostAttachment, Source
|
||||
from .cleanup_service import delete_images
|
||||
from .discord_ingester import AUTHORS_KEY
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
PLATFORM = "discord"
|
||||
# The record keys that name a message's poster. `author_name` (the display
|
||||
# name) is only on records written after 2026-09-28.
|
||||
_POSTER_KEYS = ("author_id", "author", "author_name")
|
||||
|
||||
|
||||
class PosterCleanupError(ValueError):
|
||||
"""The source can't be cleaned this way (not Discord, or no list)."""
|
||||
|
||||
|
||||
def source_authors(source: Source) -> list[str]:
|
||||
"""The source's poster list, lowercased; empty means everyone's wanted."""
|
||||
authors = (source.config_overrides or {}).get(AUTHORS_KEY) or []
|
||||
if not isinstance(authors, list):
|
||||
return []
|
||||
return sorted({str(a).strip().lower() for a in authors if str(a).strip()})
|
||||
|
||||
|
||||
def _poster(key: str):
|
||||
return func.lower(Post.raw_metadata[key].as_string())
|
||||
|
||||
|
||||
def _has_poster():
|
||||
return or_(*(Post.raw_metadata[k].as_string().isnot(None) for k in ("author_id", "author")))
|
||||
|
||||
|
||||
def _real_post_conditions(source_id: int) -> list:
|
||||
return [Post.source_id == source_id, Post.synthesized_by.is_(None)]
|
||||
|
||||
|
||||
def _named_poster_conditions(source_id: int) -> list:
|
||||
"""A real post of this source whose record says who posted it."""
|
||||
return [*_real_post_conditions(source_id), _has_poster()]
|
||||
|
||||
|
||||
def _other_poster_post_conditions(source_id: int, authors: list[str]) -> list:
|
||||
"""The posts that go: a named poster none of whose names is on the list."""
|
||||
return [
|
||||
*_named_poster_conditions(source_id),
|
||||
*(func.coalesce(_poster(k), "").notin_(authors) for k in _POSTER_KEYS),
|
||||
]
|
||||
|
||||
|
||||
def _doomed_post_ids(source_id: int, authors: list[str]):
|
||||
# correlate(None): used inside queries that are themselves over `post` (the
|
||||
# drops, the delete), where auto-correlation would bind this to the outer
|
||||
# row instead of scanning the table.
|
||||
return (
|
||||
select(Post.id)
|
||||
.where(*_other_poster_post_conditions(source_id, authors))
|
||||
.correlate(None)
|
||||
)
|
||||
|
||||
|
||||
def _image_conditions(source_id: int, authors: list[str]) -> list:
|
||||
"""Images that belong to a removed post and to no kept real post."""
|
||||
doomed = _doomed_post_ids(source_id, authors)
|
||||
keeper = aliased(Post)
|
||||
on_doomed = or_(
|
||||
ImageRecord.primary_post_id.in_(doomed),
|
||||
exists().where(
|
||||
ImageProvenance.image_record_id == ImageRecord.id,
|
||||
ImageProvenance.post_id.in_(doomed),
|
||||
),
|
||||
)
|
||||
on_kept = or_(
|
||||
and_(
|
||||
ImageRecord.primary_post_id.isnot(None),
|
||||
ImageRecord.primary_post_id.notin_(doomed),
|
||||
),
|
||||
exists().where(
|
||||
ImageProvenance.image_record_id == ImageRecord.id,
|
||||
ImageProvenance.post_id == keeper.id,
|
||||
keeper.synthesized_by.is_(None),
|
||||
keeper.id.notin_(doomed),
|
||||
),
|
||||
)
|
||||
return [on_doomed, ~on_kept]
|
||||
|
||||
|
||||
def _drop_conditions(source_id: int, authors: list[str]) -> list:
|
||||
"""The synthetic drops that absorbed a removed message."""
|
||||
member = aliased(Post)
|
||||
return [
|
||||
Post.source_id == source_id,
|
||||
Post.synthesized_by.isnot(None),
|
||||
exists().where(
|
||||
member.absorbed_by_post_id == Post.id,
|
||||
member.id.in_(_doomed_post_ids(source_id, authors)),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _attachment_conditions(source_id: int, authors: list[str]) -> list:
|
||||
return [PostAttachment.post_id.in_(_doomed_post_ids(source_id, authors))]
|
||||
|
||||
|
||||
def _load(session: Session, source_id: int) -> tuple[Source, list[str]]:
|
||||
source = session.get(Source, source_id)
|
||||
if source is None:
|
||||
raise LookupError(source_id)
|
||||
if source.platform != PLATFORM:
|
||||
raise PosterCleanupError("Only a Discord source has posters to filter by.")
|
||||
authors = source_authors(source)
|
||||
if not authors:
|
||||
raise PosterCleanupError(
|
||||
"Set this source's 'Only posts by' list first — with no list, "
|
||||
"every poster is wanted and nothing would be removed."
|
||||
)
|
||||
return source, authors
|
||||
|
||||
|
||||
def _count(session: Session, stmt) -> int:
|
||||
return session.execute(stmt).scalar_one()
|
||||
|
||||
|
||||
def preview(session: Session, *, source_id: int) -> dict:
|
||||
"""What `apply` would remove, per poster, without touching anything."""
|
||||
_, authors = _load(session, source_id)
|
||||
who = func.coalesce(
|
||||
Post.raw_metadata["author_name"].as_string(),
|
||||
Post.raw_metadata["author"].as_string(),
|
||||
Post.raw_metadata["author_id"].as_string(),
|
||||
)
|
||||
rows = session.execute(
|
||||
select(who, func.count(Post.id))
|
||||
.where(*_other_poster_post_conditions(source_id, authors))
|
||||
.group_by(who)
|
||||
.order_by(func.count(Post.id).desc())
|
||||
).all()
|
||||
posters = [{"poster": name, "posts": n} for name, n in rows]
|
||||
unknown = _count(session, select(func.count(Post.id)).where(
|
||||
*_real_post_conditions(source_id), ~_has_poster(),
|
||||
))
|
||||
return {
|
||||
"authors": authors,
|
||||
"posters": posters,
|
||||
"posts": sum(p["posts"] for p in posters),
|
||||
"images": _count(session, select(func.count(ImageRecord.id)).where(
|
||||
*_image_conditions(source_id, authors))),
|
||||
"attachments": _count(session, select(func.count(PostAttachment.id)).where(
|
||||
*_attachment_conditions(source_id, authors))),
|
||||
"drops": _count(session, select(func.count(Post.id)).where(
|
||||
*_drop_conditions(source_id, authors))),
|
||||
"unknown_posts": unknown,
|
||||
}
|
||||
|
||||
|
||||
def confirm_token(projection: dict) -> str:
|
||||
"""What the operator's apply must echo: the preview it was shown, so a
|
||||
list edited between preview and apply can't delete an unseen set."""
|
||||
return f"remove-{projection['posts']}-posts-{projection['images']}-images"
|
||||
|
||||
|
||||
def apply(session: Session, *, source_id: int, images_root: Path) -> dict:
|
||||
"""Remove them. Counts are taken before the deletes they describe — once
|
||||
the rows are gone there is nothing left to count (snippet #3087 note 2)."""
|
||||
projection = preview(session, source_id=source_id)
|
||||
_, authors = _load(session, source_id)
|
||||
|
||||
image_ids = session.execute(
|
||||
select(ImageRecord.id).where(*_image_conditions(source_id, authors))
|
||||
).scalars().all()
|
||||
# Drops first-class: found BEFORE their members go, since the link that
|
||||
# finds them is the member's absorbed_by_post_id.
|
||||
drop_ids = session.execute(
|
||||
select(Post.id).where(*_drop_conditions(source_id, authors))
|
||||
).scalars().all()
|
||||
|
||||
deleted = delete_images(session, image_ids=list(image_ids), images_root=images_root)
|
||||
# Attachments before posts: post_attachment.post_id is SET NULL, and a
|
||||
# NULL-post attachment collides on its partial unique index (see
|
||||
# cleanup_service.delete_artist_cascade).
|
||||
attachments = session.execute(
|
||||
delete(PostAttachment).where(*_attachment_conditions(source_id, authors))
|
||||
).rowcount or 0
|
||||
posts = session.execute(
|
||||
Post.__table__.delete().where(Post.id.in_(_doomed_post_ids(source_id, authors)))
|
||||
).rowcount or 0
|
||||
drops = 0
|
||||
if drop_ids:
|
||||
drops = session.execute(
|
||||
Post.__table__.delete().where(Post.id.in_(drop_ids))
|
||||
).rowcount or 0
|
||||
session.commit()
|
||||
log.info(
|
||||
"discord poster cleanup (source %s, keeping %s): %d post(s), %d image(s), "
|
||||
"%d attachment(s), %d drop(s) removed",
|
||||
source_id, authors, posts, deleted["images_deleted"], attachments, drops,
|
||||
)
|
||||
return {
|
||||
**projection,
|
||||
"posts_deleted": posts,
|
||||
"images_deleted": deleted["images_deleted"],
|
||||
"files_deleted": deleted["files_deleted"],
|
||||
"attachments_deleted": attachments,
|
||||
"drops_deleted": drops,
|
||||
}
|
||||
@@ -24,7 +24,7 @@ import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from .discord_ingester import DiscordIngester
|
||||
from .gallery_dl import DownloadResult, ErrorType
|
||||
from .download_result import DownloadResult, ErrorType
|
||||
from .ingest_core import DEFAULT_REVISIT_DAYS
|
||||
from .patreon_ingester import PatreonIngester
|
||||
from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Closing the download runs a worker restart orphaned (#4433).
|
||||
|
||||
Its own module, importing nothing but the model, because its caller is the
|
||||
worker boot hook in `celery_signals` — which every download task imports via
|
||||
`celery_app`. Living in `tasks.maintenance` put the whole maintenance import
|
||||
graph, the membership roster included, on the fetch path, which
|
||||
`test_gated_reason` forbids.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import literal, update
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
|
||||
from ..models import DownloadEvent
|
||||
|
||||
DOWNLOAD_INTERRUPTED_MESSAGE = (
|
||||
"interrupted by a worker restart — the next check picks it up where it left off"
|
||||
)
|
||||
|
||||
|
||||
def interrupt_orphaned_download_events(session, *, booted_at: datetime) -> int:
|
||||
"""Close the download events a restart orphaned, without blaming the source.
|
||||
|
||||
Called when the download lane comes up (#4433). Anything still
|
||||
pending/running from before this boot belongs to the previous process:
|
||||
a walk that outlived the 90s stop grace was SIGKILLed, and a queued or
|
||||
serialize-deferred task is held unacked until Redis redelivers it about an
|
||||
hour later. Left alone, the 30-min stall sweep would error each one and
|
||||
bump `consecutive_failures`, backing the source off as if the platform had
|
||||
failed.
|
||||
|
||||
Instead they end as `skipped` (terminal, not a failure) and the source is
|
||||
not touched: `last_checked_at` keeps its old value, so the next tick finds
|
||||
it due and the walk resumes from its checkpoint. A redelivered message
|
||||
that arrives later finds no pending event and opens a fresh one.
|
||||
|
||||
An event promoted to running after the boot has `started_at` reset to its
|
||||
real start (download_service), so it is never caught here. Does NOT commit.
|
||||
"""
|
||||
now = datetime.now(UTC)
|
||||
result = session.execute(
|
||||
update(DownloadEvent)
|
||||
.where(DownloadEvent.status.in_(["pending", "running"]))
|
||||
.where(DownloadEvent.started_at < booted_at)
|
||||
.values(
|
||||
status="skipped",
|
||||
finished_at=now,
|
||||
error=DOWNLOAD_INTERRUPTED_MESSAGE,
|
||||
metadata_=DownloadEvent.metadata_.op("||")(
|
||||
literal({"error_type": "interrupted"}, JSONB)
|
||||
),
|
||||
)
|
||||
.returning(DownloadEvent.id)
|
||||
)
|
||||
return len(result.all())
|
||||
@@ -0,0 +1,222 @@
|
||||
"""What a download walk reports, whichever backend ran it.
|
||||
|
||||
The native ingesters (Patreon, SubscribeStar, Discord) and the gallery-dl
|
||||
subprocess (HentaiFoundry) both return a `DownloadResult`, classify failures
|
||||
with the same `ErrorType`s and build `run_stats` through `make_run_stats`, so
|
||||
phase 3, the source's status chip and the Logs UI read one shape from either.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class ErrorType(StrEnum):
|
||||
AUTH_ERROR = "auth_error"
|
||||
RATE_LIMITED = "rate_limited"
|
||||
NOT_FOUND = "not_found"
|
||||
ACCESS_DENIED = "access_denied"
|
||||
NETWORK_ERROR = "network_error"
|
||||
NO_NEW_CONTENT = "no_new_content"
|
||||
TIER_LIMITED = "tier_limited"
|
||||
TIMEOUT = "timeout"
|
||||
HTTP_ERROR = "http_error"
|
||||
UNSUPPORTED_URL = "unsupported_url"
|
||||
VALIDATION_FAILED = "validation_failed"
|
||||
# Native Patreon ingester only (plan #697): a response parsed as JSON but
|
||||
# didn't match the JSON:API shape the ingester depends on (missing `data`,
|
||||
# media lacking `file_name`/`url`, etc.). Distinct from AUTH_ERROR — the fix
|
||||
# is updating the ingester's field-set/parser, not rotating credentials. The
|
||||
# contract test guards against silently shipping this.
|
||||
API_DRIFT = "api_drift"
|
||||
# Run made real progress (downloaded ≥1 file) but did not finish in the
|
||||
# subprocess budget. Distinct from UNKNOWN_ERROR — the downstream status
|
||||
# mapping classifies this as "ok" because the next tick continues.
|
||||
PARTIAL = "partial"
|
||||
UNKNOWN_ERROR = "unknown_error"
|
||||
|
||||
|
||||
@dataclass
|
||||
class DownloadResult:
|
||||
success: bool
|
||||
url: str
|
||||
artist_slug: str
|
||||
platform: str
|
||||
files_downloaded: int = 0
|
||||
files_quarantined: int = 0
|
||||
quarantined_paths: list[str] = field(default_factory=list)
|
||||
written_paths: list[str] = field(default_factory=list)
|
||||
# Native ingester only: post-only sidecar paths for posts that have NO
|
||||
# downloadable media (pure-text posts), so the importer can still upsert the
|
||||
# Post + its body. Empty on the gallery-dl path. Phase 3 imports these via
|
||||
# Importer.upsert_post_record (keyed on external_post_id → updates, never
|
||||
# doubles, the same Post a media import would create).
|
||||
post_record_paths: list[str] = field(default_factory=list)
|
||||
# Native ingester, recapture mode only (#830): (on-disk path, CDN source_url)
|
||||
# pairs for already-present media whose ImageRecord.source_filehash should be
|
||||
# backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
|
||||
# on the gallery-dl path and outside recapture.
|
||||
relink_source_paths: list[tuple[str, str, str]] = field(default_factory=list)
|
||||
# Native ingester only: one entry per `written_paths` entry, the backfill
|
||||
# cursor of the page that file came from. Phase 3 resumes the next chunk
|
||||
# there when it leaves files unimported at its deadline. Empty on gallery-dl.
|
||||
written_cursors: list[str | None] = field(default_factory=list)
|
||||
stdout: str = ""
|
||||
stderr: str = ""
|
||||
return_code: int = 0
|
||||
error_type: ErrorType | None = None
|
||||
error_message: str | None = None
|
||||
duration_seconds: float = 0.0
|
||||
started_at: str | None = None
|
||||
completed_at: str | None = None
|
||||
# Plan #704 — structured fields the NATIVE ingester populates directly (None
|
||||
# on the gallery-dl path, which keeps the regex-over-stdout route). When set,
|
||||
# phase 3 reads these instead of scraping the text it would otherwise have to
|
||||
# reconstruct: `run_stats` mirrors _compute_run_stats' shape; `cursor` is the
|
||||
# backfill checkpoint the ingester knows exactly (no parse_last_cursor).
|
||||
run_stats: dict | None = None
|
||||
cursor: str | None = None
|
||||
posts_processed: int = 0
|
||||
# Plan #708 B1 — the server's Retry-After seconds on a RATE_LIMITED result, so
|
||||
# the platform cooldown matches the hint instead of a flat default. None when
|
||||
# unknown (no header, or not a rate-limit failure).
|
||||
retry_after_seconds: float | None = None
|
||||
# Native ingester only: marks this walk's fetched media seen in its ledger.
|
||||
# Phase 3 calls it AFTER the import loop, never before — a file marked seen
|
||||
# but not yet imported is invisible to every later walk, so a run killed in
|
||||
# between orphaned it for good (TamadaHeijun's 12PCG post lost 7 of 13
|
||||
# images to a stranded run, 2026-09-24). Unmarked, the next walk finds the
|
||||
# file on disk with no ImageRecord and imports it. Takes `skip_paths`: the
|
||||
# written paths phase 3 did not import, which stay unmarked for the same
|
||||
# reason. None on gallery-dl.
|
||||
mark_seen_after_import: Callable[..., None] | None = None
|
||||
|
||||
|
||||
def extract_errors_warnings(stderr: str) -> str:
|
||||
"""Keep only the `[error]`/`[warning]` lines from a captured stderr blob.
|
||||
|
||||
A generic download-log helper (module-level so the native-ingester result
|
||||
path can shape its logs without reaching through a GalleryDLService instance).
|
||||
"""
|
||||
if not stderr:
|
||||
return ""
|
||||
kept = [
|
||||
line for line in stderr.splitlines()
|
||||
if "][error]" in line.lower() or "][warning]" in line.lower()
|
||||
]
|
||||
return "\n".join(kept)
|
||||
|
||||
|
||||
def truncate_log(text: str, max_bytes: int = 500_000) -> str:
|
||||
"""Cap a captured log to `max_bytes`, eliding the middle (head + tail kept)."""
|
||||
if not text:
|
||||
return text
|
||||
encoded = text.encode("utf-8")
|
||||
if len(encoded) <= max_bytes:
|
||||
return text
|
||||
half = max_bytes // 2
|
||||
head = encoded[:half].decode("utf-8", errors="ignore")
|
||||
tail = encoded[-half:].decode("utf-8", errors="ignore")
|
||||
head_lines = head.count("\n")
|
||||
tail_lines = tail.count("\n")
|
||||
total_lines = text.count("\n")
|
||||
elided = max(0, total_lines - head_lines - tail_lines)
|
||||
marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
|
||||
return head + marker + tail
|
||||
|
||||
|
||||
def make_run_stats(
|
||||
*,
|
||||
exit_code: int = 0,
|
||||
downloaded_count: int = 0,
|
||||
skipped_count: int = 0,
|
||||
per_item_failures: int = 0,
|
||||
warning_count: int = 0,
|
||||
tier_gated_count: int = 0,
|
||||
quarantined_count: int = 0,
|
||||
dead_lettered_count: int = 0,
|
||||
) -> dict:
|
||||
"""The canonical `run_stats` dict shape, in ONE place.
|
||||
|
||||
Both result producers — gallery-dl's `_compute_run_stats` (log scrape) and the
|
||||
native ingester's per-outcome tally (ingest_core) — build through this so the
|
||||
key set can't drift between backends. Phase 3 + the Logs UI read these keys.
|
||||
"""
|
||||
return {
|
||||
"exit_code": exit_code,
|
||||
"downloaded_count": downloaded_count,
|
||||
"skipped_count": skipped_count,
|
||||
"per_item_failures": per_item_failures,
|
||||
"warning_count": warning_count,
|
||||
"tier_gated_count": tier_gated_count,
|
||||
"quarantined_count": quarantined_count,
|
||||
"dead_lettered_count": dead_lettered_count,
|
||||
}
|
||||
|
||||
|
||||
# --- tier-gated classification, shared by BOTH backends ---------------------
|
||||
#
|
||||
# These three live together because the native ingester and the gallery-dl
|
||||
# subprocess must reach the same verdict from the same number. They did not:
|
||||
# gallery-dl classified TIER_LIMITED while ingest_core counted gated posts and
|
||||
# threw the count away, so the platforms FC owns reported a paywalled creator as
|
||||
# a silent one (#874 follow-up). One predicate, spread into both, rather than
|
||||
# the condition re-derived per backend.
|
||||
|
||||
|
||||
def classify_tier_gated(tier_gated_count: int) -> ErrorType | None:
|
||||
"""TIER_LIMITED when a walk saw tier-gated posts and nothing else failed.
|
||||
|
||||
Deliberately NOT conditioned on `downloaded == 0`. A creator whose top-tier
|
||||
posts we cannot see is tier-limited even in a week we did get their cheaper
|
||||
ones — the fact the operator needs ("there is content here you are not
|
||||
paying for") is true either way. gallery-dl has classified it this way since
|
||||
the paywall-as-"needs attention" complaint (see `_categorize_error`), and the
|
||||
native path now matches rather than inventing a stricter rule.
|
||||
|
||||
Callers must apply this only AFTER the real error categories (auth, rate
|
||||
limit, drift, …) have had their turn; tier-gating is the weakest signal and
|
||||
must never mask a genuine failure.
|
||||
"""
|
||||
return ErrorType.TIER_LIMITED if tier_gated_count else None
|
||||
|
||||
|
||||
def tier_gated_message(count: int) -> str:
|
||||
"""The one wording for the tier-gated verdict, so the two backends can't
|
||||
describe the same state differently in the Logs UI."""
|
||||
return (
|
||||
f"Subscription tier does not grant access to "
|
||||
f"{count} post{'s' if count != 1 else ''}"
|
||||
)
|
||||
|
||||
|
||||
# `Source.error_type` doubles as the failure-class chip, and a status of "ok"
|
||||
# CLEARS it (alembic 0032). TIER_LIMITED breaks that assumption: it rides an
|
||||
# otherwise-successful run, so without an exemption the chip is wiped the moment
|
||||
# it is set and `FailingSourcesCard`'s `tier_limited` palette entry can never
|
||||
# render. Informational classes are the exemption — they describe the source,
|
||||
# not a failure of the run.
|
||||
INFORMATIONAL_ERROR_TYPES = frozenset({ErrorType.TIER_LIMITED.value})
|
||||
|
||||
|
||||
def is_informational(error_type) -> bool:
|
||||
"""True for a class that reports a state rather than a failure. Accepts an
|
||||
ErrorType or the plain string persisted on Source.error_type."""
|
||||
return error_type is not None and str(error_type) in INFORMATIONAL_ERROR_TYPES
|
||||
|
||||
|
||||
def walk_completed(result: DownloadResult) -> bool:
|
||||
"""Did this walk reach the bottom cleanly?
|
||||
|
||||
The backfill lifecycle's completion test. An informational error_type still
|
||||
counts as complete: a fully-paywalled creator's backfill DID finish, and
|
||||
treating it as unfinished re-walks the same wall until the stall counter
|
||||
trips — the creator we can see least becoming the one we fetch most.
|
||||
"""
|
||||
return (
|
||||
result.success
|
||||
and result.return_code == 0
|
||||
and (result.error_type is None or is_informational(result.error_type))
|
||||
)
|
||||
@@ -3,8 +3,9 @@
|
||||
Three-phase pipeline:
|
||||
Phase 1 — DB setup (brief session): load source, mark event running,
|
||||
fetch credentials + settings.
|
||||
Phase 2 — Execute (no DB connection): call GalleryDLService; on
|
||||
Patreon campaign-ID failure, resolve vanity and retry once.
|
||||
Phase 2 — Execute (no DB connection): run the source's backend (a native
|
||||
ingester, or gallery-dl for HentaiFoundry); on a Patreon
|
||||
campaign-ID failure, resolve the vanity and retry once.
|
||||
Phase 3 — Import + persist (fresh session): attach_in_place each
|
||||
written file; clean up duplicates; persist rich metadata.
|
||||
"""
|
||||
@@ -13,6 +14,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -25,27 +27,39 @@ from sqlalchemy.orm import joinedload
|
||||
from ..models import Artist, DownloadEvent, Source
|
||||
from .credential_service import CredentialService
|
||||
from .download_backends import run_download, uses_native_ingester
|
||||
from .gallery_dl import (
|
||||
BACKFILL_CHUNK_SECONDS,
|
||||
BACKFILL_SKIP_VALUE,
|
||||
TICK_SKIP_VALUE,
|
||||
from .download_result import (
|
||||
DownloadResult,
|
||||
ErrorType,
|
||||
GalleryDLService,
|
||||
SourceConfig,
|
||||
extract_errors_warnings,
|
||||
is_informational,
|
||||
truncate_log,
|
||||
walk_completed,
|
||||
)
|
||||
from .gallery_dl import (
|
||||
BACKFILL_CHUNK_SECONDS,
|
||||
BACKFILL_SKIP_VALUE,
|
||||
TICK_SKIP_VALUE,
|
||||
GalleryDLService,
|
||||
SourceConfig,
|
||||
)
|
||||
from .importer import Importer
|
||||
from .ingest_core import DEFAULT_REVISIT_DAYS
|
||||
from .ingest_core import DEFAULT_REVISIT_DAYS, PHASE3_DEADLINE_SECONDS
|
||||
from .platforms import auth_type_for
|
||||
from .scheduler_service import set_platform_cooldown
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def credential_proven(ctx: dict, dl_result) -> bool:
|
||||
"""Did this walk prove the platform's stored credential still works?
|
||||
|
||||
A clean walk made with a credential is the evidence the Verify button looks
|
||||
for. A walk with any error type proves nothing, tier-limited included:
|
||||
content hidden behind a tier is also what an expired session looks like."""
|
||||
used = bool(ctx.get("cookies_path") or ctx.get("auth_token"))
|
||||
return used and bool(dl_result.success) and dl_result.error_type is None
|
||||
|
||||
|
||||
class DownloadService:
|
||||
"""Async orchestrator. The Celery task runs `asyncio.run(svc.download_source(N))`.
|
||||
|
||||
@@ -82,6 +96,7 @@ class DownloadService:
|
||||
|
||||
async def download_source(self, source_id: int) -> int:
|
||||
"""Returns DownloadEvent.id. Idempotent: in-flight events are returned as-is."""
|
||||
started = time.monotonic()
|
||||
setup = await self._phase1_setup(source_id)
|
||||
if setup["status"] in ("skipped", "in_flight"):
|
||||
return setup["event_id"]
|
||||
@@ -174,6 +189,7 @@ class DownloadService:
|
||||
|
||||
return await self._phase3_persist(
|
||||
ctx["event_id"], ctx, dl_result, resolved_campaign_id,
|
||||
started=started,
|
||||
)
|
||||
|
||||
async def _run_download(
|
||||
@@ -268,6 +284,8 @@ class DownloadService:
|
||||
ctx: dict,
|
||||
dl_result,
|
||||
resolved_campaign_id: str | None,
|
||||
*,
|
||||
started: float | None = None,
|
||||
) -> int:
|
||||
# Cache Patreon campaign id on the source.
|
||||
if resolved_campaign_id:
|
||||
@@ -303,7 +321,23 @@ class DownloadService:
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
for path_str in dl_result.written_paths or []:
|
||||
# Phase 3 shares the task's soft limit with the walk, and an import's
|
||||
# cost is only estimated there. Native platforms stop starting imports
|
||||
# at the deadline: what is left stays on disk with no record, which the
|
||||
# next chunk's walk imports. gallery-dl platforms have no such pickup,
|
||||
# so they import everything.
|
||||
deadline = (
|
||||
started + PHASE3_DEADLINE_SECONDS
|
||||
if started is not None and uses_native_ingester(ctx["platform"])
|
||||
else None
|
||||
)
|
||||
written_paths = list(dl_result.written_paths or [])
|
||||
deferred_from: int | None = None
|
||||
|
||||
for idx, path_str in enumerate(written_paths):
|
||||
if deadline is not None and time.monotonic() >= deadline:
|
||||
deferred_from = idx
|
||||
break
|
||||
if path_str in (dl_result.quarantined_paths or []):
|
||||
continue
|
||||
path = Path(path_str)
|
||||
@@ -348,11 +382,8 @@ class DownloadService:
|
||||
if do_embed:
|
||||
embed_image.delay(img_id)
|
||||
elif result.status == "attached":
|
||||
# Non-media or extracted archive captured as PostAttachment
|
||||
# (FC-2d-iii). The canonical copy lives in the attachments
|
||||
# store; the original download path is now redundant —
|
||||
# mirror duplicate_hash cleanup so we don't keep two copies.
|
||||
# Operator-flagged 2026-06-02 (Lustria OST zip).
|
||||
# Non-media, or an archive with no new images, captured as a
|
||||
# PostAttachment; the attachments store holds the copy.
|
||||
import_summary["attached"] += 1
|
||||
# An archive captured WITHOUT extracting any image carries a
|
||||
# reason on result.error — record it so the event explains the
|
||||
@@ -369,35 +400,13 @@ class DownloadService:
|
||||
bytes_downloaded += path.stat().st_size # noqa: ASYNC240
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
path.unlink(missing_ok=True) # noqa: ASYNC240
|
||||
except OSError:
|
||||
pass
|
||||
elif result.status == "skipped" and result.skip_reason and result.skip_reason.value in (
|
||||
"duplicate_hash", "duplicate_phash",
|
||||
):
|
||||
import_summary["skipped"] += 1
|
||||
try:
|
||||
path.unlink(missing_ok=True) # noqa: ASYNC240
|
||||
except OSError:
|
||||
pass
|
||||
elif result.status == "skipped":
|
||||
# Soft skip (too_small, too_transparent, invalid_image) —
|
||||
# the file just didn't qualify, not a download/ingest
|
||||
# failure. Don't flag the run as error; the file stays
|
||||
# on disk for operator inspection.
|
||||
# A duplicate, or a file the import filters rejected. Neither
|
||||
# is a download failure, so the run is not flagged.
|
||||
import_summary["skipped"] += 1
|
||||
elif result.status == "failed":
|
||||
# Hard failure (today only: archive probe crash/timeout).
|
||||
# The original archive sits in /images/ as an orphan; the
|
||||
# filesystem scanner would re-import and re-crash on the
|
||||
# same file, so delete the source file and surface the
|
||||
# error in import_summary. Audit 2026-06-02.
|
||||
# Hard failure: an archive probe crashed or timed out.
|
||||
import_summary["errors"] += 1
|
||||
try:
|
||||
path.unlink(missing_ok=True) # noqa: ASYNC240
|
||||
except OSError:
|
||||
pass
|
||||
elif result.status == "refreshed":
|
||||
# Currently unreachable from attach_in_place (the download
|
||||
# path never runs in deep=True mode), but the importer's
|
||||
@@ -406,6 +415,10 @@ class DownloadService:
|
||||
import_summary["attached"] += 1
|
||||
else:
|
||||
import_summary["errors"] += 1
|
||||
# Remove the download when the library no longer needs it at this
|
||||
# path: an attachment or archive now in the store, a duplicate, a
|
||||
# rejected file. One policy for every download path.
|
||||
self.importer.discard_after_attach(path, result)
|
||||
|
||||
# Post-only records: media-less posts (pure-text) the native ingester
|
||||
# captured so the artist archive is complete. Upsert each (keyed on
|
||||
@@ -427,8 +440,16 @@ class DownloadService:
|
||||
# has been through the importer. Had the run died before here they stay
|
||||
# unmarked, and the next walk imports them from disk (ingest_core).
|
||||
mark_seen = getattr(dl_result, "mark_seen_after_import", None)
|
||||
deferred = written_paths[deferred_from:] if deferred_from is not None else []
|
||||
if mark_seen is not None:
|
||||
await loop.run_in_executor(None, mark_seen)
|
||||
if deferred:
|
||||
await loop.run_in_executor(
|
||||
None, lambda: mark_seen(skip_paths=frozenset(deferred)),
|
||||
)
|
||||
else:
|
||||
await loop.run_in_executor(None, mark_seen)
|
||||
if deferred:
|
||||
self._defer_unimported(dl_result, deferred_from, len(deferred))
|
||||
|
||||
# #830 recapture: backfill source_filehash on EXISTING on-disk images so
|
||||
# their post-body inline <img src=CDN> remaps to the local copy. A
|
||||
@@ -528,8 +549,39 @@ class DownloadService:
|
||||
)
|
||||
await self._apply_backfill_lifecycle(ctx, dl_result)
|
||||
await self.async_session.commit()
|
||||
if credential_proven(ctx, dl_result):
|
||||
# The credentials page then shows when the account last worked,
|
||||
# not only when someone last pressed Verify.
|
||||
await self.cred_service.mark_verified(ctx["platform"])
|
||||
return event_id
|
||||
|
||||
@staticmethod
|
||||
def _defer_unimported(dl_result, first: int, count: int) -> None:
|
||||
"""Turn this chunk into a chunk boundary at the first file phase 3 left.
|
||||
|
||||
The next chunk resumes on the page that file came from, so its walk
|
||||
meets every left file again, on disk with no record, and imports it. A
|
||||
walk that reached its end has not finished while files wait, so a clean
|
||||
result becomes PARTIAL; a real failure keeps its own error.
|
||||
"""
|
||||
cursors = getattr(dl_result, "written_cursors", None) or []
|
||||
if first < len(cursors):
|
||||
dl_result.cursor = cursors[first]
|
||||
dl_result.phase3_imported_any = first > 0
|
||||
msg = (
|
||||
f"Imports stopped at the chunk deadline: {count} file(s) left "
|
||||
"for the next chunk"
|
||||
)
|
||||
dl_result.stdout = f"{dl_result.stdout}\n{msg}" if dl_result.stdout else msg
|
||||
log.info("%s", msg)
|
||||
if dl_result.error_type in (None, ErrorType.PARTIAL, ErrorType.TIMEOUT) or (
|
||||
is_informational(dl_result.error_type)
|
||||
):
|
||||
dl_result.success = False
|
||||
dl_result.return_code = -1
|
||||
dl_result.error_type = ErrorType.PARTIAL
|
||||
dl_result.error_message = msg
|
||||
|
||||
async def _apply_backfill_lifecycle(self, ctx: dict, dl_result) -> None:
|
||||
"""Backfill state machine (plan #693, building on the cursor of #689).
|
||||
|
||||
@@ -597,9 +649,12 @@ class DownloadService:
|
||||
new_cursor = (
|
||||
dl_result.cursor if uses_native_ingester(ctx["platform"]) else None
|
||||
)
|
||||
# Imports phase 3 finished before its deadline are progress even when
|
||||
# the cursor went back to the page holding the files it left.
|
||||
advanced = bool(
|
||||
(new_cursor and new_cursor != old_cursor)
|
||||
or dl_result.files_downloaded > 0
|
||||
or getattr(dl_result, "phase3_imported_any", False)
|
||||
)
|
||||
if advanced:
|
||||
if new_cursor:
|
||||
|
||||
@@ -23,6 +23,12 @@ log = logging.getLogger(__name__)
|
||||
|
||||
# The probe runs while the chip is drawing; names that take longer are skipped.
|
||||
_NAME_LOOKUP_SECONDS = 6.0
|
||||
# The poster picker waits on two pages of a channel's messages (#4491).
|
||||
_POSTER_LOOKUP_SECONDS = 20.0
|
||||
|
||||
# A source's poster list (discord_ingester.AUTHORS_KEY) holds user ids; this
|
||||
# keeps the name each id was picked under, for showing the list to a person.
|
||||
AUTHOR_LABELS_KEY = "discord_author_labels"
|
||||
|
||||
|
||||
class UnknownPlatformError(Exception):
|
||||
@@ -37,6 +43,15 @@ class UnknownArtistError(Exception):
|
||||
"""quick-add named an `artist_id` that does not exist."""
|
||||
|
||||
|
||||
class PosterLookupError(Exception):
|
||||
"""The Discord poster picker couldn't read the channel (no token, no
|
||||
access, not a channel page, or Discord was slow)."""
|
||||
|
||||
|
||||
class UnknownSourceError(Exception):
|
||||
"""No source follows this Discord channel or its server."""
|
||||
|
||||
|
||||
# Mirrored byte-for-byte from extension/lib/platforms.js
|
||||
# PLATFORM_ARTIST_PATTERNS. Keep these two copies in sync by hand —
|
||||
# reviewers catch drift.
|
||||
@@ -109,12 +124,24 @@ class ExtensionService:
|
||||
*,
|
||||
artist_id: int | None = None,
|
||||
artist_name: str | None = None,
|
||||
use_platform_name: bool = False,
|
||||
discord_authors: list[dict] | None = None,
|
||||
) -> dict:
|
||||
"""Add `url` as a source. `artist_id` connects it to an existing
|
||||
artist, `artist_name` to that artist (created if new); with neither,
|
||||
the artist is resolved from the platform as before."""
|
||||
the artist is resolved from the platform as before.
|
||||
|
||||
`use_platform_name` applies the operator's convention that the Patreon
|
||||
name is canon: a Patreon source added to an existing artist renames
|
||||
that artist to the creator's Patreon display name. Name only — the
|
||||
slug, and every path keyed off it, never moves (#130). Ignored on every
|
||||
other platform, and when the name can't be read.
|
||||
|
||||
`discord_authors` ([{id, name}], #4491) sets a Discord source's poster
|
||||
list in the same step, on a new source or one already there."""
|
||||
platform, raw_slug = self._derive(url)
|
||||
url = canonical_source_url(platform, url, raw_slug)
|
||||
renamed_from = None
|
||||
# Identity by SOURCE handle (#130): an existing (platform, url) source
|
||||
# keeps its artist on re-add — even if that artist was since renamed (its
|
||||
# frozen slug no longer matches the current name), and even when the
|
||||
@@ -125,6 +152,8 @@ class ExtensionService:
|
||||
artist = (await self.session.execute(
|
||||
select(Artist).where(Artist.id == existing.artist_id)
|
||||
)).scalar_one()
|
||||
if platform == DISCORD and discord_authors is not None:
|
||||
await self._apply_posters(existing, discord_authors)
|
||||
return self._shape(existing, artist, created_source=False, created_artist=False)
|
||||
|
||||
if artist_id is not None:
|
||||
@@ -134,6 +163,8 @@ class ExtensionService:
|
||||
if artist is None:
|
||||
raise UnknownArtistError(f"no artist with id {artist_id}")
|
||||
created_artist = False
|
||||
if use_platform_name and platform == "patreon":
|
||||
renamed_from = await self._adopt_patreon_name(artist, raw_slug, url)
|
||||
else:
|
||||
name = (artist_name or "").strip()
|
||||
if not name:
|
||||
@@ -144,7 +175,108 @@ class ExtensionService:
|
||||
source, created_source = await self._find_or_create_source(
|
||||
artist_id=artist.id, platform=platform, url=url,
|
||||
)
|
||||
return self._shape(source, artist, created_source, created_artist)
|
||||
if platform == DISCORD and discord_authors is not None:
|
||||
await self._apply_posters(source, discord_authors)
|
||||
shaped = self._shape(source, artist, created_source, created_artist)
|
||||
if renamed_from is not None:
|
||||
shaped["renamed_from"] = renamed_from
|
||||
return shaped
|
||||
|
||||
# -- Discord poster picker (#4491) ------------------------------------------
|
||||
|
||||
def _discord_page(self, url: str) -> tuple[str, str]:
|
||||
platform, raw_slug = self._derive(url)
|
||||
if platform != DISCORD:
|
||||
raise InvalidUrlError("not a Discord page")
|
||||
server_id, _, channel_id = raw_slug.partition("/")
|
||||
if not channel_id:
|
||||
raise PosterLookupError("Open a channel to see who posts in it.")
|
||||
return server_id, channel_id
|
||||
|
||||
async def _source_for_page(self, server_id: str, channel_id: str) -> Source | None:
|
||||
"""The source this channel's posts land on: the channel's own, else a
|
||||
whole-server source that walks it."""
|
||||
base = f"https://discord.com/channels/{server_id}"
|
||||
return (
|
||||
await self._existing_source(DISCORD, f"{base}/{channel_id}")
|
||||
or await self._existing_source(DISCORD, base)
|
||||
)
|
||||
|
||||
async def discord_posters(self, url: str) -> dict:
|
||||
"""Who posts in the channel `url` shows, newest `_POSTER_SCAN`
|
||||
messages deep, with the likely creator marked — and, when a source
|
||||
already follows it, the poster list that source has now."""
|
||||
server_id, channel_id = self._discord_page(url)
|
||||
import asyncio
|
||||
|
||||
from .credential_service import CredentialService
|
||||
from .discord_client import DiscordAPIError, DiscordClient
|
||||
token = None
|
||||
if self._crypto is not None:
|
||||
token = await CredentialService(self.session, self._crypto).get_token(DISCORD)
|
||||
if not token:
|
||||
raise PosterLookupError("No Discord token is saved in FabledCurator.")
|
||||
client = DiscordClient(token, max_retries=0)
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
found = await asyncio.wait_for(
|
||||
loop.run_in_executor(None, client.recent_posters, server_id, channel_id),
|
||||
timeout=_POSTER_LOOKUP_SECONDS,
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
raise PosterLookupError("Discord took too long to answer; try again.") from exc
|
||||
except DiscordAPIError as exc:
|
||||
raise PosterLookupError(f"Couldn't read this channel: {exc}") from exc
|
||||
source = await self._source_for_page(server_id, channel_id)
|
||||
co = (source.config_overrides or {}) if source is not None else {}
|
||||
from .discord_ingester import AUTHORS_KEY
|
||||
return {
|
||||
**found,
|
||||
"source_id": source.id if source is not None else None,
|
||||
"selected": list(co.get(AUTHORS_KEY) or []),
|
||||
# So someone on the list who hasn't posted lately still shows by
|
||||
# name, and can be unticked.
|
||||
"selected_labels": dict(co.get(AUTHOR_LABELS_KEY) or {}),
|
||||
}
|
||||
|
||||
async def set_discord_posters(self, url: str, authors: list[dict]) -> dict:
|
||||
"""Store the picked posters on the source that follows `url`'s channel."""
|
||||
server_id, channel_id = self._discord_page(url)
|
||||
source = await self._source_for_page(server_id, channel_id)
|
||||
if source is None:
|
||||
raise UnknownSourceError("No source follows this channel yet.")
|
||||
return await self._apply_posters(source, authors)
|
||||
|
||||
async def _apply_posters(self, source: Source, authors: list[dict]) -> dict:
|
||||
"""Ids are what the list keys on — a username can change on a whim,
|
||||
an id can't — and the name each was picked under rides beside it for
|
||||
showing the list to a person. An empty pick clears the list, which
|
||||
takes everyone's posts again."""
|
||||
from .discord_ingester import AUTHORS_KEY
|
||||
picked = [a for a in authors if str(a.get("id") or "").strip()]
|
||||
co = dict(source.config_overrides or {})
|
||||
if picked:
|
||||
co[AUTHORS_KEY] = [str(a["id"]).strip() for a in picked]
|
||||
co[AUTHOR_LABELS_KEY] = {
|
||||
str(a["id"]).strip(): str(a.get("name") or a["id"]) for a in picked
|
||||
}
|
||||
else:
|
||||
co.pop(AUTHORS_KEY, None)
|
||||
co.pop(AUTHOR_LABELS_KEY, None)
|
||||
source.config_overrides = co or None
|
||||
await self.session.commit()
|
||||
return {"source_id": source.id, "discord_authors": co.get(AUTHORS_KEY, [])}
|
||||
|
||||
async def _adopt_patreon_name(self, artist, raw_slug: str, url: str) -> str | None:
|
||||
"""Rename `artist` to the Patreon display name; the old name when it
|
||||
changed, else None. Unreadable name → no rename, never the handle."""
|
||||
name = await self._platform_display_name("patreon", raw_slug, url)
|
||||
if not name or name == artist.name:
|
||||
return None
|
||||
old = artist.name
|
||||
artist.name = name
|
||||
await self.session.commit()
|
||||
return old
|
||||
|
||||
async def _existing_source(self, platform: str, url: str) -> Source | None:
|
||||
"""The source this URL already is, whichever artist owns it. Discord
|
||||
@@ -193,8 +325,18 @@ class ExtensionService:
|
||||
server_id = raw_slug.split("/", 1)[0]
|
||||
names = await self._discord_names(server_id, None)
|
||||
return names.get("server") or f"Discord {server_id}"
|
||||
return await self._platform_display_name(platform, raw_slug, url) or raw_slug
|
||||
|
||||
async def _platform_display_name(
|
||||
self, platform: str, raw_slug: str, url: str
|
||||
) -> str | None:
|
||||
"""The creator's display name as Patreon or SubscribeStar shows it, read
|
||||
with the stored cookies; None when it can't be read (no credential, a
|
||||
network error, a slow answer, any other platform). None, not the handle,
|
||||
so a caller can tell a real name from a fallback — a rename to the
|
||||
Patreon name must never rename to a URL handle instead."""
|
||||
if self._crypto is None or platform not in ("patreon", "subscribestar"):
|
||||
return raw_slug
|
||||
return None
|
||||
import asyncio
|
||||
|
||||
from .credential_service import CredentialService
|
||||
@@ -204,7 +346,7 @@ class ExtensionService:
|
||||
if platform == "patreon":
|
||||
cookies = await cred.get_cookies_path("patreon")
|
||||
from .patreon_resolver import resolve_display_name
|
||||
name = await loop.run_in_executor(
|
||||
call = loop.run_in_executor(
|
||||
None, resolve_display_name, raw_slug,
|
||||
str(cookies) if cookies else None,
|
||||
)
|
||||
@@ -212,15 +354,14 @@ class ExtensionService:
|
||||
cookies = await cred.get_cookies_path("subscribestar")
|
||||
from .subscribestar_client import SubscribeStarClient
|
||||
client = SubscribeStarClient(str(cookies) if cookies else None)
|
||||
name = await loop.run_in_executor(
|
||||
None, client.resolve_display_name, url
|
||||
)
|
||||
call = loop.run_in_executor(None, client.resolve_display_name, url)
|
||||
name = await asyncio.wait_for(call, timeout=_NAME_LOOKUP_SECONDS)
|
||||
except Exception as exc: # resolution is best-effort — never block the add
|
||||
log.warning("artist display-name resolution failed (%s): %s", platform, exc)
|
||||
return raw_slug
|
||||
return name or raw_slug
|
||||
return None
|
||||
return (name or "").strip() or None
|
||||
|
||||
async def probe(self, url: str) -> dict:
|
||||
async def probe(self, url: str, *, names: bool = False) -> dict:
|
||||
"""Read-only resolution of a creator-page URL against the FC DB.
|
||||
Returns one of:
|
||||
- {state: 'unknown_platform'} — URL didn't match any
|
||||
@@ -236,7 +377,11 @@ class ExtensionService:
|
||||
— exact (artist, platform,
|
||||
url) Source already exists
|
||||
|
||||
Side-effect-free: two SELECTs at most.
|
||||
`names` (the Add panel asks, the chip does not) adds `display_name`:
|
||||
the creator's name as Patreon/SubscribeStar shows it, or None. It costs
|
||||
a request to the platform, so a plain page view never pays it.
|
||||
|
||||
Side-effect-free: two SELECTs at most, plus that one lookup.
|
||||
"""
|
||||
try:
|
||||
platform, raw_slug = self._derive(url)
|
||||
@@ -246,13 +391,16 @@ class ExtensionService:
|
||||
return await self._probe_discord(raw_slug)
|
||||
|
||||
slug = slugify(raw_slug)
|
||||
result: dict = {"platform": platform, "slug": slug}
|
||||
if names:
|
||||
result["display_name"] = await self._platform_display_name(
|
||||
platform, raw_slug, url,
|
||||
)
|
||||
artist = (await self.session.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
)).scalar_one_or_none()
|
||||
if artist is None:
|
||||
return {"state": "new", "platform": platform, "slug": slug}
|
||||
|
||||
artist_payload = {"id": artist.id, "name": artist.name, "slug": artist.slug}
|
||||
return {"state": "new", **result}
|
||||
|
||||
source = (await self.session.execute(
|
||||
select(Source).where(
|
||||
@@ -262,25 +410,13 @@ class ExtensionService:
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
if source is None:
|
||||
return {
|
||||
"state": "artist_match",
|
||||
"platform": platform,
|
||||
"slug": slug,
|
||||
"artist": artist_payload,
|
||||
}
|
||||
return {"state": "artist_match", **result, "artist": self._artist_payload(artist)}
|
||||
|
||||
return {
|
||||
"state": "source_match",
|
||||
"platform": platform,
|
||||
"slug": slug,
|
||||
"artist": artist_payload,
|
||||
"source": {
|
||||
"id": source.id,
|
||||
"artist_id": source.artist_id,
|
||||
"platform": source.platform,
|
||||
"url": source.url,
|
||||
"enabled": source.enabled,
|
||||
},
|
||||
**result,
|
||||
"artist": self._artist_payload(artist),
|
||||
"source": self._source_payload(source),
|
||||
}
|
||||
|
||||
async def _probe_discord(self, raw_slug: str) -> dict:
|
||||
|
||||
@@ -241,11 +241,19 @@ def _fetch_gdrive(url: str, dest_dir: Path, *, read_timeout: float,
|
||||
def _fetch_mega(url: str, dest_dir: Path, *, read_timeout: float,
|
||||
total_timeout: float,
|
||||
should_stop: Callable[[], bool]) -> FetchResult:
|
||||
before = set(dest_dir.iterdir()) if dest_dir.exists() else set()
|
||||
# A folder link keeps its subfolders, so files are found at any depth.
|
||||
# Dotfiles are megatools' `.megatmp.*` staging names.
|
||||
def _complete_files() -> set[Path]:
|
||||
return {
|
||||
p for p in dest_dir.rglob("*")
|
||||
if p.is_file() and not p.name.startswith(".")
|
||||
} if dest_dir.exists() else set()
|
||||
|
||||
before = _complete_files()
|
||||
# megatools is a subprocess: its timeout IS a total wall-clock cap (the read
|
||||
# timeout has no analogue here), so the total budget applies directly.
|
||||
_run_mega_get(url, str(dest_dir), timeout=total_timeout)
|
||||
new = [p for p in dest_dir.iterdir() if p not in before and p.is_file()]
|
||||
new = sorted(_complete_files() - before)
|
||||
if not new:
|
||||
return FetchResult(error="mega-get wrote no new file")
|
||||
return FetchResult(files=new, bytes=sum(p.stat().st_size for p in new))
|
||||
|
||||
@@ -21,9 +21,9 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def refetch_links_for_post(session: Session, post_id: int) -> dict:
|
||||
"""Reset every settled ExternalLink on a post to pending (fresh attempt
|
||||
budget) and dispatch its fetch. In-flight ('downloading') links are left
|
||||
alone. Commits. Returns {links_total, links_reset}."""
|
||||
"""Reset every settled ExternalLink on a post to pending (fresh attempt and
|
||||
recovery budgets) and dispatch its fetch. In-flight ('downloading') links
|
||||
are left alone. Commits. Returns {links_total, links_reset}."""
|
||||
links = session.execute(
|
||||
select(ExternalLink).where(ExternalLink.post_id == post_id)
|
||||
).scalars().all()
|
||||
@@ -33,6 +33,7 @@ def refetch_links_for_post(session: Session, post_id: int) -> dict:
|
||||
continue
|
||||
link.status = "pending"
|
||||
link.attempts = 0
|
||||
link.recoveries = 0
|
||||
link.last_error = None
|
||||
link.completed_at = None
|
||||
reset_ids.append(link.id)
|
||||
|
||||
@@ -6,6 +6,9 @@ backend/app/services/gallery_dl.py). FC adaptations:
|
||||
- Validation uses backend/app/services/file_validator.py
|
||||
- Config dir under `<images_root>/.gallery-dl/`, not a separate config_path
|
||||
- PLATFORM_DEFAULTS scoped to the GS-6 platforms registered in FC-3b
|
||||
|
||||
The result it returns (`DownloadResult`, `ErrorType`) is shared with the native
|
||||
ingesters and lives in download_result.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -17,42 +20,22 @@ import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
from pathlib import Path
|
||||
|
||||
from .download_result import (
|
||||
DownloadResult,
|
||||
ErrorType,
|
||||
classify_tier_gated,
|
||||
make_run_stats,
|
||||
tier_gated_message,
|
||||
)
|
||||
from .file_validator import is_validatable, quarantine_file, validate_file
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ErrorType(StrEnum):
|
||||
AUTH_ERROR = "auth_error"
|
||||
RATE_LIMITED = "rate_limited"
|
||||
NOT_FOUND = "not_found"
|
||||
ACCESS_DENIED = "access_denied"
|
||||
NETWORK_ERROR = "network_error"
|
||||
NO_NEW_CONTENT = "no_new_content"
|
||||
TIER_LIMITED = "tier_limited"
|
||||
TIMEOUT = "timeout"
|
||||
HTTP_ERROR = "http_error"
|
||||
UNSUPPORTED_URL = "unsupported_url"
|
||||
VALIDATION_FAILED = "validation_failed"
|
||||
# Native Patreon ingester only (plan #697): a response parsed as JSON but
|
||||
# didn't match the JSON:API shape the ingester depends on (missing `data`,
|
||||
# media lacking `file_name`/`url`, etc.). Distinct from AUTH_ERROR — the fix
|
||||
# is updating the ingester's field-set/parser, not rotating credentials. The
|
||||
# contract test guards against silently shipping this.
|
||||
API_DRIFT = "api_drift"
|
||||
# Run made real progress (downloaded ≥1 file) but did not finish in the
|
||||
# subprocess budget. Distinct from UNKNOWN_ERROR — the downstream status
|
||||
# mapping classifies this as "ok" because the next tick continues.
|
||||
PARTIAL = "partial"
|
||||
UNKNOWN_ERROR = "unknown_error"
|
||||
|
||||
|
||||
# Tick mode (routine cron polls): skip ≤20 contiguous already-archived
|
||||
# items, then exit gallery-dl. Established subscription with zero new
|
||||
# content exits in ~30s of HEAD requests instead of walking to the bottom
|
||||
@@ -148,89 +131,6 @@ class SourceConfig:
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DownloadResult:
|
||||
success: bool
|
||||
url: str
|
||||
artist_slug: str
|
||||
platform: str
|
||||
files_downloaded: int = 0
|
||||
files_quarantined: int = 0
|
||||
quarantined_paths: list[str] = field(default_factory=list)
|
||||
written_paths: list[str] = field(default_factory=list)
|
||||
# Native ingester only: post-only sidecar paths for posts that have NO
|
||||
# downloadable media (pure-text posts), so the importer can still upsert the
|
||||
# Post + its body. Empty on the gallery-dl path. Phase 3 imports these via
|
||||
# Importer.upsert_post_record (keyed on external_post_id → updates, never
|
||||
# doubles, the same Post a media import would create).
|
||||
post_record_paths: list[str] = field(default_factory=list)
|
||||
# Native ingester, recapture mode only (#830): (on-disk path, CDN source_url)
|
||||
# pairs for already-present media whose ImageRecord.source_filehash should be
|
||||
# backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
|
||||
# on the gallery-dl path and outside recapture.
|
||||
relink_source_paths: list[tuple[str, str, str]] = field(default_factory=list)
|
||||
stdout: str = ""
|
||||
stderr: str = ""
|
||||
return_code: int = 0
|
||||
error_type: ErrorType | None = None
|
||||
error_message: str | None = None
|
||||
duration_seconds: float = 0.0
|
||||
started_at: str | None = None
|
||||
completed_at: str | None = None
|
||||
# Plan #704 — structured fields the NATIVE ingester populates directly (None
|
||||
# on the gallery-dl path, which keeps the regex-over-stdout route). When set,
|
||||
# phase 3 reads these instead of scraping the text it would otherwise have to
|
||||
# reconstruct: `run_stats` mirrors _compute_run_stats' shape; `cursor` is the
|
||||
# backfill checkpoint the ingester knows exactly (no parse_last_cursor).
|
||||
run_stats: dict | None = None
|
||||
cursor: str | None = None
|
||||
posts_processed: int = 0
|
||||
# Plan #708 B1 — the server's Retry-After seconds on a RATE_LIMITED result, so
|
||||
# the platform cooldown matches the hint instead of a flat default. None when
|
||||
# unknown (no header, or not a rate-limit failure).
|
||||
retry_after_seconds: float | None = None
|
||||
# Native ingester only: marks this walk's fetched media seen in its ledger.
|
||||
# Phase 3 calls it AFTER the import loop, never before — a file marked seen
|
||||
# but not yet imported is invisible to every later walk, so a run killed in
|
||||
# between orphaned it for good (TamadaHeijun's 12PCG post lost 7 of 13
|
||||
# images to a stranded run, 2026-09-24). Unmarked, the next walk finds the
|
||||
# file on disk with no ImageRecord and imports it. None on gallery-dl.
|
||||
mark_seen_after_import: Callable[[], None] | None = None
|
||||
|
||||
|
||||
def extract_errors_warnings(stderr: str) -> str:
|
||||
"""Keep only the `[error]`/`[warning]` lines from a captured stderr blob.
|
||||
|
||||
A generic download-log helper (module-level so the native-ingester result
|
||||
path can shape its logs without reaching through a GalleryDLService instance).
|
||||
"""
|
||||
if not stderr:
|
||||
return ""
|
||||
kept = [
|
||||
line for line in stderr.splitlines()
|
||||
if "][error]" in line.lower() or "][warning]" in line.lower()
|
||||
]
|
||||
return "\n".join(kept)
|
||||
|
||||
|
||||
def truncate_log(text: str, max_bytes: int = 500_000) -> str:
|
||||
"""Cap a captured log to `max_bytes`, eliding the middle (head + tail kept)."""
|
||||
if not text:
|
||||
return text
|
||||
encoded = text.encode("utf-8")
|
||||
if len(encoded) <= max_bytes:
|
||||
return text
|
||||
half = max_bytes // 2
|
||||
head = encoded[:half].decode("utf-8", errors="ignore")
|
||||
tail = encoded[-half:].decode("utf-8", errors="ignore")
|
||||
head_lines = head.count("\n")
|
||||
tail_lines = tail.count("\n")
|
||||
total_lines = text.count("\n")
|
||||
elided = max(0, total_lines - head_lines - tail_lines)
|
||||
marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
|
||||
return head + marker + tail
|
||||
|
||||
|
||||
def _summarize_validation_failures(failures: list[dict]) -> str:
|
||||
if not failures:
|
||||
return "Validation failed"
|
||||
@@ -245,106 +145,6 @@ def _summarize_validation_failures(failures: list[dict]) -> str:
|
||||
return f"{n} files quarantined ({top_count}× {top_reason}, mixed)"
|
||||
|
||||
|
||||
# (parse_last_cursor was removed in plan #704: the native ingester now carries
|
||||
# its checkpoint cursor as a structured DownloadResult.cursor field, so there is
|
||||
# no log text to scrape — and gallery-dl platforms never had a cursor.)
|
||||
|
||||
|
||||
def make_run_stats(
|
||||
*,
|
||||
exit_code: int = 0,
|
||||
downloaded_count: int = 0,
|
||||
skipped_count: int = 0,
|
||||
per_item_failures: int = 0,
|
||||
warning_count: int = 0,
|
||||
tier_gated_count: int = 0,
|
||||
quarantined_count: int = 0,
|
||||
dead_lettered_count: int = 0,
|
||||
) -> dict:
|
||||
"""The canonical `run_stats` dict shape, in ONE place.
|
||||
|
||||
Both result producers — gallery-dl's `_compute_run_stats` (log scrape) and the
|
||||
native ingester's per-outcome tally (ingest_core) — build through this so the
|
||||
key set can't drift between backends. Phase 3 + the Logs UI read these keys.
|
||||
"""
|
||||
return {
|
||||
"exit_code": exit_code,
|
||||
"downloaded_count": downloaded_count,
|
||||
"skipped_count": skipped_count,
|
||||
"per_item_failures": per_item_failures,
|
||||
"warning_count": warning_count,
|
||||
"tier_gated_count": tier_gated_count,
|
||||
"quarantined_count": quarantined_count,
|
||||
"dead_lettered_count": dead_lettered_count,
|
||||
}
|
||||
|
||||
|
||||
# --- tier-gated classification, shared by BOTH backends ---------------------
|
||||
#
|
||||
# These three live together because the native ingester and the gallery-dl
|
||||
# subprocess must reach the same verdict from the same number. They did not:
|
||||
# gallery-dl classified TIER_LIMITED while ingest_core counted gated posts and
|
||||
# threw the count away, so the platforms FC owns reported a paywalled creator as
|
||||
# a silent one (#874 follow-up). One predicate, spread into both, rather than
|
||||
# the condition re-derived per backend.
|
||||
|
||||
|
||||
def classify_tier_gated(tier_gated_count: int) -> ErrorType | None:
|
||||
"""TIER_LIMITED when a walk saw tier-gated posts and nothing else failed.
|
||||
|
||||
Deliberately NOT conditioned on `downloaded == 0`. A creator whose top-tier
|
||||
posts we cannot see is tier-limited even in a week we did get their cheaper
|
||||
ones — the fact the operator needs ("there is content here you are not
|
||||
paying for") is true either way. gallery-dl has classified it this way since
|
||||
the paywall-as-"needs attention" complaint (see `_categorize_error`), and the
|
||||
native path now matches rather than inventing a stricter rule.
|
||||
|
||||
Callers must apply this only AFTER the real error categories (auth, rate
|
||||
limit, drift, …) have had their turn; tier-gating is the weakest signal and
|
||||
must never mask a genuine failure.
|
||||
"""
|
||||
return ErrorType.TIER_LIMITED if tier_gated_count else None
|
||||
|
||||
|
||||
def tier_gated_message(count: int) -> str:
|
||||
"""The one wording for the tier-gated verdict, so the two backends can't
|
||||
describe the same state differently in the Logs UI."""
|
||||
return (
|
||||
f"Subscription tier does not grant access to "
|
||||
f"{count} post{'s' if count != 1 else ''}"
|
||||
)
|
||||
|
||||
|
||||
# `Source.error_type` doubles as the failure-class chip, and a status of "ok"
|
||||
# CLEARS it (alembic 0032). TIER_LIMITED breaks that assumption: it rides an
|
||||
# otherwise-successful run, so without an exemption the chip is wiped the moment
|
||||
# it is set and `FailingSourcesCard`'s `tier_limited` palette entry can never
|
||||
# render. Informational classes are the exemption — they describe the source,
|
||||
# not a failure of the run.
|
||||
INFORMATIONAL_ERROR_TYPES = frozenset({ErrorType.TIER_LIMITED.value})
|
||||
|
||||
|
||||
def is_informational(error_type) -> bool:
|
||||
"""True for a class that reports a state rather than a failure. Accepts an
|
||||
ErrorType or the plain string persisted on Source.error_type."""
|
||||
return error_type is not None and str(error_type) in INFORMATIONAL_ERROR_TYPES
|
||||
|
||||
|
||||
def walk_completed(result: DownloadResult) -> bool:
|
||||
"""Did this walk reach the bottom cleanly?
|
||||
|
||||
The backfill lifecycle's completion test. An informational error_type still
|
||||
counts as complete: a fully-paywalled creator's backfill DID finish, and
|
||||
treating it as unfinished re-walks the same wall until the stall counter
|
||||
trips — the creator we can see least becoming the one we fetch most.
|
||||
"""
|
||||
return (
|
||||
result.success
|
||||
and result.return_code == 0
|
||||
and (result.error_type is None or is_informational(result.error_type))
|
||||
)
|
||||
|
||||
|
||||
class GalleryDLService:
|
||||
"""Service for executing gallery-dl downloads."""
|
||||
|
||||
@@ -722,16 +522,6 @@ class GalleryDLService:
|
||||
tier_gated_count=tier_gated_count,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_errors_warnings(stderr: str) -> str:
|
||||
# Thin delegator to the module-level helper (kept for existing callers).
|
||||
return extract_errors_warnings(stderr)
|
||||
|
||||
@staticmethod
|
||||
def _truncate_log(text: str, max_bytes: int = 500_000) -> str:
|
||||
# Thin delegator to the module-level helper (kept for existing callers).
|
||||
return truncate_log(text, max_bytes)
|
||||
|
||||
async def download(
|
||||
self,
|
||||
url: str,
|
||||
|
||||
+325
-175
@@ -16,6 +16,7 @@ from dataclasses import dataclass, field
|
||||
from enum import StrEnum
|
||||
from pathlib import Path
|
||||
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
from PIL import Image
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
@@ -43,16 +44,15 @@ from ..utils.paths import (
|
||||
from ..utils.phash import compute_phash, find_similar, fingerprint_path, fingerprints_match
|
||||
from ..utils.sidecar import find_sidecar, parse_sidecar
|
||||
from ..utils.slug import slugify
|
||||
from .archive_extractor import extract_archive, is_archive
|
||||
from ..utils.staging import is_temp_staging_file
|
||||
from .archive_extractor import extract_archive, is_archive, software_marker
|
||||
from .attachment_store import AttachmentStore
|
||||
from .audits import single_color
|
||||
from .link_extract import extract_external_links
|
||||
from .thumbnailer import Thumbnailer
|
||||
from .wip_title import (
|
||||
WIP_TITLE_SOFT_SOURCE,
|
||||
WIP_TITLE_SOURCE,
|
||||
apply_wip_image_tags,
|
||||
matches_soft_wip_title,
|
||||
matches_wip_title,
|
||||
resolve_wip_tag_id,
|
||||
)
|
||||
@@ -117,6 +117,21 @@ def is_video(path: Path) -> bool:
|
||||
return path.suffix.lower() in VIDEO_EXTS
|
||||
|
||||
|
||||
def _not_content(path: Path) -> ImportResult | None:
|
||||
"""The skip result for a file that is never library content: a sidecar
|
||||
`.json` (metadata) or a downloader's staging file (an unfinished download,
|
||||
which becomes content only under its final name). None for anything else."""
|
||||
if path.suffix.lower() == ".json":
|
||||
error = "sidecar json is metadata, not content"
|
||||
elif is_temp_staging_file(path):
|
||||
error = "unfinished download (staging file), not content"
|
||||
else:
|
||||
return None
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.invalid_image, error=error,
|
||||
)
|
||||
|
||||
|
||||
def _safe_ext(path: Path) -> str:
|
||||
"""Conservatively extract a file extension for PostAttachment.ext
|
||||
(varchar(32)). Thin wrapper over the shared `utils.paths.safe_ext` (kept for
|
||||
@@ -293,6 +308,44 @@ class Importer:
|
||||
sp.rollback()
|
||||
return self.session.execute(stmt).scalar_one()
|
||||
|
||||
def _store_new_record(self, record: ImageRecord) -> ImageRecord | None:
|
||||
"""Insert `record`, or return the row that already holds its bytes.
|
||||
|
||||
The sha256 lookup before an insert can lose a race: two copies of one
|
||||
picture in different folders, imported side by side by two workers,
|
||||
both find no row, and the unique index refuses the second insert. Only
|
||||
that insert is undone (a savepoint), so the session stays usable and
|
||||
the caller files its file as a duplicate of the winner. None means
|
||||
`record` was stored."""
|
||||
sp = self.session.begin_nested()
|
||||
try:
|
||||
self.session.add(record)
|
||||
self.session.flush()
|
||||
sp.commit()
|
||||
return None
|
||||
except IntegrityError:
|
||||
sp.rollback()
|
||||
winner = self.session.execute(
|
||||
select(ImageRecord).where(ImageRecord.sha256 == record.sha256)
|
||||
).scalar_one_or_none()
|
||||
if winner is None:
|
||||
raise
|
||||
return winner
|
||||
|
||||
def _file_as_duplicate(
|
||||
self, existing: ImageRecord, path: Path, artist: Artist | None,
|
||||
explicit_source: Source | None,
|
||||
) -> ImportResult:
|
||||
"""A byte-identical copy of `existing`: link the copy's post to the
|
||||
image (enrich-on-duplicate), so a picture posted twice shows on both
|
||||
posts, and report the skip."""
|
||||
self._apply_sidecar(existing, path, artist, explicit_source=explicit_source)
|
||||
self.session.commit()
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.duplicate_hash,
|
||||
image_id=existing.id, error="sha256 already present",
|
||||
)
|
||||
|
||||
def _find_or_create_source(
|
||||
self, *, artist_id: int, platform: str, url: str,
|
||||
) -> Source:
|
||||
@@ -412,11 +465,8 @@ class Importer:
|
||||
media members (one Post via the archive-adjacent sidecar) and
|
||||
preserve the archive itself. Other non-media → PostAttachment.
|
||||
Nothing is skipped for being non-media (FC-2d-iii)."""
|
||||
if source.suffix.lower() == ".json":
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||
error="sidecar json is metadata, not content",
|
||||
)
|
||||
if (refused := _not_content(source)) is not None:
|
||||
return refused
|
||||
if is_archive(source):
|
||||
return self._import_archive(source)
|
||||
if not is_supported(source):
|
||||
@@ -428,11 +478,19 @@ class Importer:
|
||||
return self._upsert_artist(name) if name else None
|
||||
|
||||
def _post_for_sidecar(
|
||||
self, source: Path, artist: Artist | None
|
||||
self, source: Path, artist: Artist | None,
|
||||
*, source_row: Source | None = None,
|
||||
) -> Post | None:
|
||||
"""If a sidecar sits next to `source`, ensure its Source+Post
|
||||
exist (idempotent) and return the Post — so attachments can link
|
||||
to the same Post the per-member _apply_sidecar will reuse."""
|
||||
to the same Post the per-member _apply_sidecar will reuse.
|
||||
|
||||
`source_row` is the subscription being downloaded, when there is one,
|
||||
and wins over the (artist, platform) lookup — as it does in
|
||||
`upsert_post_record`. The lookup takes the artist's FIRST source on the
|
||||
platform, which is right only while an artist has one: a Discord artist
|
||||
has one per channel, and every non-image file from a later channel was
|
||||
filed under the first as an undated second post (#4435)."""
|
||||
sc = find_sidecar(source)
|
||||
if sc is None or artist is None:
|
||||
return None
|
||||
@@ -444,10 +502,13 @@ class Importer:
|
||||
log.warning("sidecar parse failed for %s: %s", sc, exc)
|
||||
return None
|
||||
sd = parse_sidecar(data)
|
||||
platform = sd.platform or "unknown"
|
||||
src = self._lookup_source_for_sidecar(
|
||||
artist_id=artist.id, platform=platform,
|
||||
)
|
||||
if source_row is not None:
|
||||
src = source_row
|
||||
else:
|
||||
platform = sd.platform or "unknown"
|
||||
src = self._lookup_source_for_sidecar(
|
||||
artist_id=artist.id, platform=platform,
|
||||
)
|
||||
epid = sd.external_post_id or sc.stem
|
||||
return self._find_or_create_post(
|
||||
source_id=src.id if src else None,
|
||||
@@ -533,28 +594,42 @@ class Importer:
|
||||
# lays files out under /images/<artist_slug>/...). Filesystem
|
||||
# import still calls bare _import_archive(source) and falls
|
||||
# back to the path-walk derivation as before.
|
||||
probe = safe_probe.probe_archive(source)
|
||||
#
|
||||
# A game or app build is recognised first, from the archive's directory
|
||||
# alone. The probe reads and checksums every member, which for a
|
||||
# multi-gigabyte build is minutes of work on a file that is kept whole
|
||||
# and never extracted anyway.
|
||||
marker = software_marker(source)
|
||||
probe = None if marker is not None else safe_probe.probe_archive(source)
|
||||
if probe is not None and probe.crashed:
|
||||
return ImportResult(
|
||||
status="failed",
|
||||
error=f"archive probe crashed/timed out: {probe.reason}",
|
||||
)
|
||||
|
||||
artist_use = artist if artist is not None else self._resolve_artist(source)
|
||||
post = self._post_for_sidecar(source, artist_use, source_row=source_row)
|
||||
if marker is not None:
|
||||
# A game or app build: kept whole, its bundled images left alone.
|
||||
self._capture_attachment(
|
||||
source, post=post, artist=artist_use, resolved=True,
|
||||
)
|
||||
log.info("%s: a software build (holds %s), kept unextracted",
|
||||
source.name, marker)
|
||||
return ImportResult(status="attached")
|
||||
if not probe.ok:
|
||||
if probe.crashed:
|
||||
return ImportResult(
|
||||
status="failed",
|
||||
error=f"archive probe crashed/timed out: {probe.reason}",
|
||||
)
|
||||
# Clean rejection (bomb cap exceeded, integrity mismatch):
|
||||
# still preserve the archive file itself as an attachment so
|
||||
# nothing silently vanishes, matching extract_archive's
|
||||
# fail-soft contract.
|
||||
artist_use = artist if artist is not None else self._resolve_artist(source)
|
||||
post = self._post_for_sidecar(source, artist_use)
|
||||
# Clean rejection (bomb cap exceeded, integrity mismatch): the
|
||||
# archive file itself is still kept as an attachment so nothing
|
||||
# silently vanishes, matching extract_archive's fail-soft contract.
|
||||
self._capture_attachment(
|
||||
source, post=post, artist=artist_use, resolved=True,
|
||||
)
|
||||
reason = f"archive probe rejected, captured unextracted: {probe.reason}"
|
||||
log.warning("%s: %s", source.name, reason)
|
||||
return ImportResult(status="attached", error=reason)
|
||||
|
||||
artist_use = artist if artist is not None else self._resolve_artist(source)
|
||||
post = self._post_for_sidecar(source, artist_use)
|
||||
# Committed before extraction, so a member that fails (and is rolled
|
||||
# back on its own) can never take the archive's post with it.
|
||||
self.session.commit()
|
||||
member_ids: list[int] = []
|
||||
# Every member image touched (new + superseded + deduped), so the
|
||||
# from_attachment_id stamp below covers files that already existed in the
|
||||
@@ -567,7 +642,7 @@ class Importer:
|
||||
# or failed/corrupt members — instead of one catch-all string.
|
||||
counts = {
|
||||
"media": 0, "deduped": 0, "unsupported": 0,
|
||||
"failed": 0, "nested": 0, "nested_rejected": 0,
|
||||
"failed": 0, "nested": 0, "nested_rejected": 0, "nested_software": 0,
|
||||
}
|
||||
self._collect_archive_members(
|
||||
source, attribution=source, source_row=source_row,
|
||||
@@ -599,20 +674,21 @@ class Importer:
|
||||
if counts["media"] == 0 and counts["nested"] == 0:
|
||||
reason = ("archive yielded no members (unsupported/corrupt, or the "
|
||||
"extractor backend failed)")
|
||||
elif counts["deduped"] and not (
|
||||
elif (counts["deduped"] or counts["nested_software"]) and not (
|
||||
counts["unsupported"] or counts["failed"] or counts["nested_rejected"]
|
||||
):
|
||||
log.info(
|
||||
"%s: all %d image member(s) already in library (deduped + linked "
|
||||
"to this post) — archive preserved, no new import",
|
||||
source.name, counts["deduped"],
|
||||
"%s: no new images — %d already in the library (linked to this "
|
||||
"post), %d nested software build(s) skipped; archive preserved",
|
||||
source.name, counts["deduped"], counts["nested_software"],
|
||||
)
|
||||
return ImportResult(status="attached")
|
||||
else:
|
||||
reason = (
|
||||
f"no new images — media={counts['media']} deduped={counts['deduped']} "
|
||||
f"unsupported/non-media={counts['unsupported']} failed={counts['failed']} "
|
||||
f"nested={counts['nested']} nested-rejected={counts['nested_rejected']}"
|
||||
f"nested={counts['nested']} nested-rejected={counts['nested_rejected']} "
|
||||
f"nested-software={counts['nested_software']}"
|
||||
)
|
||||
log.warning("%s: %s", source.name, reason)
|
||||
return ImportResult(status="attached", error=reason)
|
||||
@@ -645,6 +721,15 @@ class Importer:
|
||||
_ARCHIVE_MAX_DEPTH, member_path.name,
|
||||
)
|
||||
continue
|
||||
# Recognised from its directory before the probe reads
|
||||
# every member, as for the outer archive.
|
||||
if (marker := software_marker(member_path)) is not None:
|
||||
counts["nested_software"] += 1
|
||||
log.info(
|
||||
"nested archive is a software build (holds %s), "
|
||||
"skipped: %s", marker, member_path.name,
|
||||
)
|
||||
continue
|
||||
probe = safe_probe.probe_archive(member_path)
|
||||
if not probe.ok:
|
||||
counts["nested_rejected"] += 1
|
||||
@@ -681,7 +766,15 @@ class Importer:
|
||||
member_record_ids.add(res.image_id)
|
||||
else:
|
||||
counts["failed"] += 1
|
||||
except SoftTimeLimitExceeded:
|
||||
# The task's time budget is spent: stop here so the task marks
|
||||
# its row failed, rather than carrying on until the hard kill.
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 — defensive per level; keep going
|
||||
# Each member commits on its own, so this undoes only the one in
|
||||
# progress; without it the next query fails on the dead transaction
|
||||
# and the archive is never stored.
|
||||
self.session.rollback()
|
||||
log.warning(
|
||||
"archive extraction failed for %s (depth %d): %s",
|
||||
archive_path.name, depth, exc,
|
||||
@@ -781,7 +874,6 @@ class Importer:
|
||||
# Compute file dimensions (images only) and apply filters.
|
||||
width = height = None
|
||||
duration = None # video container duration (Tier-1 near-dup key, #871)
|
||||
has_alpha = False
|
||||
if is_video(source):
|
||||
# Layer-3 isolation: validate the container via ffprobe (a
|
||||
# separate process) before the rest of the pipeline touches
|
||||
@@ -800,52 +892,9 @@ class Importer:
|
||||
)
|
||||
width, height, duration = probe.width, probe.height, probe.duration
|
||||
else:
|
||||
try:
|
||||
with Image.open(source) as im:
|
||||
im.verify()
|
||||
with Image.open(source) as im:
|
||||
width, height = im.size
|
||||
has_alpha = im.mode in ("RGBA", "LA") or (
|
||||
im.mode == "P" and "transparency" in im.info
|
||||
)
|
||||
except Exception as exc:
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.invalid_image, error=str(exc),
|
||||
)
|
||||
|
||||
if (self.settings.min_width and width < self.settings.min_width) or (
|
||||
self.settings.min_height and height < self.settings.min_height
|
||||
):
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.too_small,
|
||||
error=f"{width}x{height} below min {self.settings.min_width}x{self.settings.min_height}",
|
||||
)
|
||||
|
||||
if self.settings.skip_transparent and has_alpha:
|
||||
try:
|
||||
pct = self._transparency_pct(source)
|
||||
except OSError as exc:
|
||||
# PIL.verify() at line 263 only validates header structure;
|
||||
# truncated/corrupt pixel data only surfaces when load()
|
||||
# actually decodes (here via getchannel('A')). Convert to
|
||||
# invalid_image skip so the Celery autoretry loop doesn't
|
||||
# bounce the same broken file forever.
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||
error=f"PIL load failed during transparency check: {exc}",
|
||||
)
|
||||
if pct >= self.settings.transparency_threshold:
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.too_transparent,
|
||||
error=f"{pct:.2%} transparent",
|
||||
)
|
||||
|
||||
if self.settings.skip_single_color and self._single_color_hit(source):
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.single_color,
|
||||
error=(f"one color dominates >"
|
||||
f"{self.settings.single_color_threshold:.0%}"),
|
||||
)
|
||||
refusal, width, height = self._screen_still(source)
|
||||
if refusal is not None:
|
||||
return refusal
|
||||
|
||||
# Artist anchored to the attribution path (folder→artist), resolved
|
||||
# UP-FRONT so the enrich-on-duplicate branches link provenance with the
|
||||
@@ -862,34 +911,17 @@ class Importer:
|
||||
existing = self.session.execute(existing_stmt).scalar_one_or_none()
|
||||
if existing:
|
||||
if not self.deep:
|
||||
# Enrich-on-duplicate (parity with attach_in_place): a re-scanned
|
||||
# file that is a byte-dup of an existing image still links its
|
||||
# post via _apply_sidecar, so a cross-posted image shows on every
|
||||
# post.
|
||||
self._apply_sidecar(
|
||||
existing, attribution_path, path_artist,
|
||||
explicit_source=explicit_source,
|
||||
)
|
||||
self.session.commit()
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.duplicate_hash,
|
||||
image_id=existing.id, error="sha256 already present",
|
||||
return self._file_as_duplicate(
|
||||
existing, attribution_path, path_artist, explicit_source,
|
||||
)
|
||||
return self._deep_rederive(existing, source, attribution_path)
|
||||
|
||||
# Perceptual near-dup (images only; videos keep phash NULL).
|
||||
phash = None
|
||||
if not is_video(source):
|
||||
try:
|
||||
with Image.open(source) as im:
|
||||
phash = compute_phash(im)
|
||||
except OSError as exc:
|
||||
# Same rationale as the transparency-check guard above:
|
||||
# broken-pixel-data files pass verify() but blow up here.
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||
error=f"PIL load failed during phash compute: {exc}",
|
||||
)
|
||||
refusal, phash = self._phash_of(source)
|
||||
if refusal is not None:
|
||||
return refusal
|
||||
if phash is not None:
|
||||
candidates = self._phash_candidates_cache()
|
||||
rel, match_id = find_similar(
|
||||
@@ -959,8 +991,14 @@ class Importer:
|
||||
origin="imported_filesystem",
|
||||
integrity_status="unknown",
|
||||
)
|
||||
self.session.add(record)
|
||||
self.session.flush()
|
||||
if (winner := self._store_new_record(record)) is not None:
|
||||
# The copy just written is a stray unless it is the winner's own
|
||||
# file (the same source imported twice lands on the same path).
|
||||
if Path(winner.path) != dest:
|
||||
dest.unlink(missing_ok=True)
|
||||
return self._file_as_duplicate(
|
||||
winner, attribution_path, path_artist, explicit_source,
|
||||
)
|
||||
self._phash_cache_append(phash, width, height, record.id)
|
||||
|
||||
# Folder→artist (path_artist resolved up-front above); bind it to the
|
||||
@@ -1040,9 +1078,7 @@ class Importer:
|
||||
removal sticks. The existing catalogue is covered separately by the
|
||||
operator-triggered backfill sweep. Gated by the settings toggle, and
|
||||
best-effort: any failure is logged, never allowed to fail the import."""
|
||||
hard_on = self.settings.wip_title_tagging_enabled
|
||||
soft_on = self.settings.wip_soft_title_tagging_enabled
|
||||
if not (hard_on or soft_on):
|
||||
if not self.settings.wip_title_tagging_enabled:
|
||||
return
|
||||
if record.primary_post_id is None:
|
||||
return
|
||||
@@ -1050,21 +1086,14 @@ class Importer:
|
||||
title = self.session.execute(
|
||||
select(Post.post_title).where(Post.id == record.primary_post_id)
|
||||
).scalar_one_or_none()
|
||||
# HARD tier ("WIP"/"work in progress") wins — higher precision, and it
|
||||
# trains the head; SOFT (sketch/doodle, #1474) is the provisional fallback
|
||||
# that never trains (source wip_title_soft).
|
||||
if hard_on and matches_wip_title(title):
|
||||
source = WIP_TITLE_SOURCE
|
||||
elif soft_on and matches_soft_wip_title(title):
|
||||
source = WIP_TITLE_SOFT_SOURCE
|
||||
else:
|
||||
if not matches_wip_title(title):
|
||||
return
|
||||
if self._wip_tag_id is _UNSET:
|
||||
self._wip_tag_id = resolve_wip_tag_id(self.session)
|
||||
if self._wip_tag_id is None:
|
||||
return
|
||||
apply_wip_image_tags(
|
||||
self.session, [record.id], self._wip_tag_id, source=source
|
||||
self.session, [record.id], self._wip_tag_id, source=WIP_TITLE_SOURCE
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — a tag must never fail an import
|
||||
log.warning(
|
||||
@@ -1141,9 +1170,51 @@ class Importer:
|
||||
if post.artist_id is None:
|
||||
post.artist_id = artist.id
|
||||
self._apply_post_fields(post, sd)
|
||||
self._redate_post_images(post)
|
||||
self.session.commit()
|
||||
return True
|
||||
|
||||
def _redate_post_images(self, post: Post) -> None:
|
||||
"""Carry a post's date onto the images already linked to it (#4431).
|
||||
|
||||
The native ingesters import a post's media BEFORE its record: the
|
||||
per-media sidecar holds only the image identity (post-first, #856), and
|
||||
the date arrives with `_post.json`. So `_attach_provenance` links each
|
||||
image to a post that has no date yet, and the image keeps its download
|
||||
time. This runs when the record lands, and applies the same two rules
|
||||
`_attach_provenance` applies: `effective_date` is the PRIMARY post's
|
||||
date, and `earliest_post_date` is the earliest date across every post
|
||||
the image is linked to. Only rows that differ are written."""
|
||||
if post.post_date is None:
|
||||
return
|
||||
self.session.flush()
|
||||
self.session.execute(
|
||||
update(ImageRecord)
|
||||
.where(ImageRecord.primary_post_id == post.id)
|
||||
.where(ImageRecord.effective_date.is_distinct_from(post.post_date))
|
||||
.values(effective_date=post.post_date)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
linked = select(ImageProvenance.image_record_id).where(
|
||||
ImageProvenance.post_id == post.id
|
||||
)
|
||||
earliest = (
|
||||
select(func.min(Post.post_date))
|
||||
.select_from(ImageProvenance)
|
||||
.join(Post, Post.id == ImageProvenance.post_id)
|
||||
.where(ImageProvenance.image_record_id == ImageRecord.id)
|
||||
.where(Post.post_date.is_not(None))
|
||||
.correlate(ImageRecord)
|
||||
.scalar_subquery()
|
||||
)
|
||||
self.session.execute(
|
||||
update(ImageRecord)
|
||||
.where(ImageRecord.id.in_(linked))
|
||||
.where(ImageRecord.earliest_post_date.is_distinct_from(earliest))
|
||||
.values(earliest_post_date=earliest)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
|
||||
def attach_in_place(
|
||||
self,
|
||||
path: Path,
|
||||
@@ -1177,62 +1248,28 @@ class Importer:
|
||||
caller may delete the on-disk original (mirrors duplicate_hash)
|
||||
- failed → file untouched; caller decides
|
||||
"""
|
||||
if path.suffix.lower() == ".json":
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||
error="sidecar json is metadata, not content",
|
||||
)
|
||||
if (refused := _not_content(path)) is not None:
|
||||
return refused
|
||||
if is_archive(path):
|
||||
return self._import_archive(
|
||||
path, artist=artist, source_row=source,
|
||||
)
|
||||
if not is_supported(path):
|
||||
post = self._post_for_sidecar(path, artist) if artist else None
|
||||
post = (
|
||||
self._post_for_sidecar(path, artist, source_row=source)
|
||||
if artist else None
|
||||
)
|
||||
return self._capture_attachment(
|
||||
path, post=post, artist=artist, resolved=True,
|
||||
)
|
||||
|
||||
# Format / dimension / transparency filters (mirror _import_media).
|
||||
# The same content filters as _import_media.
|
||||
width = height = None
|
||||
duration = None # video container duration (Tier-1 near-dup key, #871)
|
||||
has_alpha = False
|
||||
if not is_video(path):
|
||||
try:
|
||||
with Image.open(path) as im:
|
||||
im.verify()
|
||||
with Image.open(path) as im:
|
||||
width, height = im.size
|
||||
has_alpha = im.mode in ("RGBA", "LA") or (
|
||||
im.mode == "P" and "transparency" in im.info
|
||||
)
|
||||
except Exception as exc:
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
if (self.settings.min_width and width < self.settings.min_width) or (
|
||||
self.settings.min_height and height < self.settings.min_height
|
||||
):
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.too_small,
|
||||
error=f"{width}x{height} below min {self.settings.min_width}x{self.settings.min_height}",
|
||||
)
|
||||
|
||||
if self.settings.skip_transparent and has_alpha:
|
||||
pct = self._transparency_pct(path)
|
||||
if pct >= self.settings.transparency_threshold:
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.too_transparent,
|
||||
error=f"{pct:.2%} transparent",
|
||||
)
|
||||
|
||||
if self.settings.skip_single_color and self._single_color_hit(path):
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.single_color,
|
||||
error=(f"one color dominates >"
|
||||
f"{self.settings.single_color_threshold:.0%}"),
|
||||
)
|
||||
refusal, width, height = self._screen_still(path)
|
||||
if refusal is not None:
|
||||
return refusal
|
||||
else:
|
||||
# Best-effort probe for dims + duration so downloaded videos can dedup
|
||||
# (#871). LENIENT: unlike _import_media this path does not reject on a
|
||||
@@ -1255,21 +1292,14 @@ class Importer:
|
||||
# only sets it when NULL. Without this the new post is left with no
|
||||
# image AND (if its other content also deduped) no attachment, i.e. a
|
||||
# bare shell — operator-flagged 2026-06-08 (1589 empty Anduo posts).
|
||||
self._apply_sidecar(existing, path, artist, explicit_source=source)
|
||||
self.session.commit()
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.duplicate_hash,
|
||||
image_id=existing.id, error="sha256 already present",
|
||||
)
|
||||
return self._file_as_duplicate(existing, path, artist, source)
|
||||
|
||||
# phash dedup + supersede
|
||||
phash = None
|
||||
if not is_video(path):
|
||||
try:
|
||||
with Image.open(path) as im:
|
||||
phash = compute_phash(im)
|
||||
except Exception:
|
||||
phash = None
|
||||
refusal, phash = self._phash_of(path)
|
||||
if refusal is not None:
|
||||
return refusal
|
||||
if phash is not None:
|
||||
candidates = self._phash_candidates_cache()
|
||||
rel, match_id = find_similar(
|
||||
@@ -1340,8 +1370,8 @@ class Importer:
|
||||
)
|
||||
if artist is not None:
|
||||
record.artist_id = artist.id
|
||||
self.session.add(record)
|
||||
self.session.flush()
|
||||
if (winner := self._store_new_record(record)) is not None:
|
||||
return self._file_as_duplicate(winner, path, artist, source)
|
||||
self._phash_cache_append(phash, width, height, record.id)
|
||||
|
||||
# Sidecar provenance (best-effort). When `source` is passed, link
|
||||
@@ -1356,6 +1386,51 @@ class Importer:
|
||||
self.session.commit()
|
||||
return ImportResult(status="imported", image_id=record.id)
|
||||
|
||||
def discard_after_attach(self, path: Path, result: ImportResult) -> bool:
|
||||
"""Delete a downloaded file that `attach_in_place` has finished with,
|
||||
when keeping it would only leave a second copy or a rejected file on
|
||||
disk. Returns True when the file was removed. Both download paths (a
|
||||
subscription walk's import step and the external-link worker) apply
|
||||
this one policy:
|
||||
|
||||
- attached: the attachments store holds the canonical copy.
|
||||
- imported or superseded ARCHIVE: its new images are in the library and
|
||||
the archive itself is in the attachments store.
|
||||
- duplicate skip: the library already holds the content.
|
||||
- soft skip (too small, too transparent, single colour, not a readable
|
||||
image): the import filters rejected it. A file left on disk with no
|
||||
record is picked up by the next walk and imported again; the seen
|
||||
ledger is what stops it being downloaded again.
|
||||
- failed: the archive probe crashed, and a retry would crash again.
|
||||
|
||||
An imported or superseded media file is kept: it IS the record's file.
|
||||
So is any path an ImageRecord points at, whatever the status, because a
|
||||
re-fetch can write identical bytes over a record's own file. A staging
|
||||
file is kept too: it may be a download still being written, and the
|
||||
temp-file sweep removes it once it is old enough to be abandoned."""
|
||||
if is_temp_staging_file(path):
|
||||
return False
|
||||
if result.status in ("imported", "superseded"):
|
||||
redundant = is_archive(path)
|
||||
else:
|
||||
redundant = result.status in ("attached", "skipped", "failed")
|
||||
if not redundant:
|
||||
return False
|
||||
owned = self.session.execute(
|
||||
select(ImageRecord.id).where(
|
||||
ImageRecord.path.in_({str(path), str(path.resolve())})
|
||||
).limit(1)
|
||||
).first()
|
||||
if owned is not None:
|
||||
log.info("keeping %s: it is image %s's file", path.name, owned.id)
|
||||
return False
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
except OSError as exc:
|
||||
log.warning("could not remove %s: %s", path, exc)
|
||||
return False
|
||||
return True
|
||||
|
||||
def _upsert_artist(self, name: str) -> Artist:
|
||||
slug = slugify(name)
|
||||
artist = self.session.execute(
|
||||
@@ -1733,6 +1808,81 @@ class Importer:
|
||||
# is operator-tunable per surface.
|
||||
_SINGLE_COLOR_TOLERANCE = 30
|
||||
|
||||
def _screen_still(
|
||||
self, source: Path,
|
||||
) -> tuple[ImportResult | None, int | None, int | None]:
|
||||
"""Read a still image's size and apply the library's content filters.
|
||||
|
||||
Returns `(refusal, width, height)`. `refusal` is a skip result when
|
||||
the file is not a readable image, is below the minimum size, is mostly
|
||||
transparent or is one colour, and None when the file passes.
|
||||
|
||||
A file whose header verifies but whose pixel data is truncated only
|
||||
fails once PIL decodes it, as the transparency check does. That decode
|
||||
error is a skip too: raised, it would retry the import task forever,
|
||||
or abort a download run in the middle of its files.
|
||||
"""
|
||||
try:
|
||||
with Image.open(source) as im:
|
||||
im.verify()
|
||||
with Image.open(source) as im:
|
||||
width, height = im.size
|
||||
has_alpha = im.mode in ("RGBA", "LA") or (
|
||||
im.mode == "P" and "transparency" in im.info
|
||||
)
|
||||
except Exception as exc:
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||
error=str(exc),
|
||||
), None, None
|
||||
|
||||
if (self.settings.min_width and width < self.settings.min_width) or (
|
||||
self.settings.min_height and height < self.settings.min_height
|
||||
):
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.too_small,
|
||||
error=f"{width}x{height} below min {self.settings.min_width}x{self.settings.min_height}",
|
||||
), width, height
|
||||
|
||||
if self.settings.skip_transparent and has_alpha:
|
||||
try:
|
||||
pct = self._transparency_pct(source)
|
||||
except OSError as exc:
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||
error=f"PIL load failed during transparency check: {exc}",
|
||||
), width, height
|
||||
if pct >= self.settings.transparency_threshold:
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.too_transparent,
|
||||
error=f"{pct:.2%} transparent",
|
||||
), width, height
|
||||
|
||||
if self.settings.skip_single_color and self._single_color_hit(source):
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.single_color,
|
||||
error=(f"one color dominates >"
|
||||
f"{self.settings.single_color_threshold:.0%}"),
|
||||
), width, height
|
||||
|
||||
return None, width, height
|
||||
|
||||
def _phash_of(self, source: Path) -> tuple[ImportResult | None, object]:
|
||||
"""The perceptual hash of a still image, as `(refusal, phash)`.
|
||||
|
||||
Hashing decodes every pixel, so this is where a file with truncated
|
||||
pixel data can first fail; that is a skip rather than an exception,
|
||||
for the same reason as in `_screen_still`.
|
||||
"""
|
||||
try:
|
||||
with Image.open(source) as im:
|
||||
return None, compute_phash(im)
|
||||
except OSError as exc:
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||
error=f"PIL load failed during phash compute: {exc}",
|
||||
), None
|
||||
|
||||
def _single_color_hit(self, source: Path) -> bool:
|
||||
"""True when one color dominates beyond the configured threshold — the
|
||||
same canonical predicate the Cleanup audit runs (audits.single_color,
|
||||
|
||||
@@ -37,7 +37,7 @@ from sqlalchemy import delete, func, select, text
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from ..models import ImageRecord
|
||||
from .gallery_dl import (
|
||||
from .download_result import (
|
||||
DownloadResult,
|
||||
ErrorType,
|
||||
classify_tier_gated,
|
||||
@@ -114,13 +114,29 @@ _CANARY_MIN_SAMPLE = 30
|
||||
# on-disk files, and died at the soft limit.
|
||||
#
|
||||
# So the walk also stops when its elapsed time PLUS phase 3's estimated cost
|
||||
# would pass CHUNK_TOTAL_SECONDS. Costs measured on the live instance: 431
|
||||
# imports took 976s (~2.3s each: hash, pHash, sidecar, provenance); a relink is
|
||||
# a sha256 over NFS, 0.15s for the 8.8 MB average file, plus a lookup.
|
||||
# test_download_source_task pins CHUNK_TOTAL_SECONDS under the soft limit.
|
||||
# would pass CHUNK_TOTAL_SECONDS. An import (hash, pHash, sidecar, provenance,
|
||||
# thumbnail enqueue) runs 2-5s depending on the files; the estimate takes the
|
||||
# slow end. A relink is a sha256 over NFS plus a lookup.
|
||||
#
|
||||
# The estimate is a plan, not a guard. PHASE3_DEADLINE_SECONDS is the guard:
|
||||
# phase 3 starts no import past it (download_service), and leaves the rest on
|
||||
# disk, unmarked, for the next chunk. It sits under the soft limit by enough to
|
||||
# finish the import in hand, upsert post records and finalize the event.
|
||||
# test_download_source_task pins both under the soft limit.
|
||||
CHUNK_TOTAL_SECONDS = 1200.0
|
||||
PHASE3_IMPORT_SECONDS = 2.5
|
||||
PHASE3_IMPORT_SECONDS = 5.0
|
||||
PHASE3_RELINK_SECONDS = 0.25
|
||||
PHASE3_DEADLINE_SECONDS = 1200.0
|
||||
|
||||
|
||||
def _unskipped(
|
||||
fetched: list[tuple[str, str]],
|
||||
written_keys: dict[str, tuple[str, str]],
|
||||
skip_paths,
|
||||
) -> list[tuple[str, str]]:
|
||||
"""`fetched` without the ledger keys of files phase 3 left unimported."""
|
||||
skip = {written_keys[p] for p in skip_paths if p in written_keys}
|
||||
return [kv for kv in fetched if kv not in skip] if skip else list(fetched)
|
||||
|
||||
|
||||
def _parse_published(raw: object) -> datetime | None:
|
||||
@@ -289,6 +305,17 @@ class Ingester:
|
||||
# Media handed to phase 3 for import. Marked seen by phase 3 once the
|
||||
# import has run (`mark_seen_after_import`), not here — see there.
|
||||
fetched: list[tuple[str, str]] = []
|
||||
# Per entry of `written`: the cursor of the page it came from, and its
|
||||
# ledger key. Phase 3 can stop importing at its deadline; the files it
|
||||
# leaves must stay unmarked, and the next chunk has to resume on the
|
||||
# page holding the first of them, or no walk sees them again.
|
||||
written_cursors: list[str | None] = []
|
||||
written_keys: dict[str, tuple[str, str]] = {}
|
||||
# Post-record keys written this walk. Marked with the media, after
|
||||
# phase 3 has upserted the records — marking them at write time left a
|
||||
# walk killed before phase 3 with posts the ledger calls recorded that
|
||||
# the database never dated, and no later tick walks back to them (#4436).
|
||||
recorded: list[tuple[str, str]] = []
|
||||
downloaded = 0
|
||||
errors = 0
|
||||
quarantined = 0
|
||||
@@ -342,7 +369,11 @@ class Ingester:
|
||||
written_paths=written,
|
||||
post_record_paths=list(post_records),
|
||||
relink_source_paths=list(relink),
|
||||
mark_seen_after_import=lambda: self._mark_seen(source_id, fetched),
|
||||
written_cursors=list(written_cursors),
|
||||
mark_seen_after_import=lambda skip_paths=(): self._mark_seen(
|
||||
source_id,
|
||||
_unskipped(fetched, written_keys, skip_paths) + recorded,
|
||||
),
|
||||
stdout="\n".join(log_lines),
|
||||
stderr="",
|
||||
return_code=return_code,
|
||||
@@ -511,7 +542,7 @@ class Ingester:
|
||||
posts_with_body += 1
|
||||
if rec.path is not None:
|
||||
post_records.append(str(rec.path))
|
||||
self._mark_seen(source_id, [(pkey, ppid)])
|
||||
recorded.append((pkey, ppid))
|
||||
# Per-post handling line in the run stdout (the existing
|
||||
# "Raw stdout" panel) — the downloader already read the
|
||||
# post; we only format its outcome here. post_type beside
|
||||
@@ -570,6 +601,8 @@ class Ingester:
|
||||
downloaded += 1
|
||||
if outcome.path is not None:
|
||||
written.append(str(outcome.path))
|
||||
written_cursors.append(emitted_cursor)
|
||||
written_keys[str(outcome.path)] = (key, media_item.post_id)
|
||||
fetched.append((key, media_item.post_id))
|
||||
to_clear.append(key)
|
||||
consecutive_seen = 0
|
||||
@@ -583,6 +616,8 @@ class Ingester:
|
||||
# attach_in_place because no record owns this path —
|
||||
# the unlink below is about a file that IS the record.
|
||||
written.append(str(outcome.path))
|
||||
written_cursors.append(emitted_cursor)
|
||||
written_keys[str(outcome.path)] = (key, media_item.post_id)
|
||||
fetched.append((key, media_item.post_id))
|
||||
to_clear.append(key)
|
||||
skipped_count += 1
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Files in the artists' folders that the library has no record of.
|
||||
|
||||
A download writes its files into the artist's folder and imports them in a
|
||||
later step. A run that stops between the two (a restart, a time limit) leaves
|
||||
complete files on disk that no image or attachment points at, and a walk that
|
||||
has already marked them seen never comes back for them. `find_unimported`
|
||||
lists those files; `queue_unimported` turns the same list into an import batch
|
||||
that attaches each file where it lies (`scan_mode="library"`), so a file the
|
||||
library already holds, or one the import filters reject, is removed instead.
|
||||
|
||||
Only files under a folder named for an existing artist (matched by slug, so a
|
||||
differently-cased folder still matches) are considered, and only images,
|
||||
videos and archives. Files changed in the last day are left for the download
|
||||
that may still be writing them.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import stat
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import Artist, ImageRecord, ImportBatch, ImportTask, PostAttachment, Source
|
||||
from ..utils.slug import slugify
|
||||
from ..utils.staging import is_temp_staging_file
|
||||
from .archive_extractor import is_archive
|
||||
from .importer import is_supported
|
||||
|
||||
# Top-level folders that hold the library's own stores, not an artist's files.
|
||||
_STORE_DIRS = frozenset({"thumbs", "attachments", "cookies", "secrets"})
|
||||
MIN_AGE_SECONDS = 24 * 3600
|
||||
_ACTIVE_TASK_STATUSES = ("pending", "queued", "processing")
|
||||
_SAMPLES_PER_ARTIST = 3
|
||||
_ARTISTS_LISTED = 50
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Unimported:
|
||||
path: Path
|
||||
artist_slug: str
|
||||
task_type: str # "media" or "archive", as ImportTask.task_type
|
||||
size: int
|
||||
|
||||
|
||||
def _referenced_paths(session: Session) -> set[str]:
|
||||
paths: set[str] = set()
|
||||
for p, thumb in session.execute(select(ImageRecord.path, ImageRecord.thumbnail_path)):
|
||||
paths.add(p)
|
||||
if thumb:
|
||||
paths.add(thumb)
|
||||
paths.update(session.execute(select(PostAttachment.path)).scalars())
|
||||
paths.update(session.execute(
|
||||
select(ImportTask.source_path).where(ImportTask.status.in_(_ACTIVE_TASK_STATUSES))
|
||||
).scalars())
|
||||
return paths
|
||||
|
||||
|
||||
def find_unimported(
|
||||
session: Session, images_root: Path, *, now: float | None = None,
|
||||
) -> tuple[list[Unimported], dict]:
|
||||
"""Walk the artists' folders and return (files to import, summary).
|
||||
|
||||
The summary counts what was found and what was passed over, and lists the
|
||||
artists with the most to import (bytes first) with a few sample paths each:
|
||||
{"files", "bytes", "archives", "artists": [{slug, files, bytes,
|
||||
samples}], "recent", "not_media", "no_artist"}"""
|
||||
now = time.time() if now is None else now
|
||||
referenced = _referenced_paths(session)
|
||||
slugs = set(session.execute(select(Artist.slug)).scalars())
|
||||
found: list[Unimported] = []
|
||||
passed = {"recent": 0, "not_media": 0, "no_artist": 0}
|
||||
root = Path(images_root)
|
||||
for top in sorted(os.listdir(root)):
|
||||
if top in _STORE_DIRS or top.startswith(("_", ".")):
|
||||
continue
|
||||
top_dir = root / top
|
||||
if not top_dir.is_dir():
|
||||
continue
|
||||
slug = slugify(top)
|
||||
known = slug in slugs
|
||||
for dirpath, dirnames, filenames in os.walk(top_dir):
|
||||
dirnames[:] = sorted(d for d in dirnames if not d.startswith("."))
|
||||
for name in sorted(filenames):
|
||||
path = Path(dirpath) / name
|
||||
if (
|
||||
name.startswith(".") or path.suffix.lower() == ".json"
|
||||
or is_temp_staging_file(path)
|
||||
):
|
||||
continue
|
||||
if str(path) in referenced:
|
||||
continue
|
||||
try:
|
||||
st = path.lstat()
|
||||
except OSError:
|
||||
continue
|
||||
if not stat.S_ISREG(st.st_mode):
|
||||
continue
|
||||
kind = "archive" if is_archive(path) else ("media" if is_supported(path) else None)
|
||||
if kind is None:
|
||||
passed["not_media"] += 1
|
||||
elif now - st.st_mtime < MIN_AGE_SECONDS:
|
||||
passed["recent"] += 1
|
||||
elif not known:
|
||||
passed["no_artist"] += 1
|
||||
else:
|
||||
found.append(Unimported(path, slug, kind, st.st_size))
|
||||
|
||||
per_artist: dict[str, dict] = defaultdict(lambda: {"files": 0, "bytes": 0, "samples": []})
|
||||
for u in found:
|
||||
a = per_artist[u.artist_slug]
|
||||
a["files"] += 1
|
||||
a["bytes"] += u.size
|
||||
if len(a["samples"]) < _SAMPLES_PER_ARTIST:
|
||||
a["samples"].append(str(u.path.relative_to(root)))
|
||||
artists = sorted(
|
||||
({"slug": s, **v} for s, v in per_artist.items()), key=lambda a: -a["bytes"],
|
||||
)[:_ARTISTS_LISTED]
|
||||
summary = {
|
||||
"files": len(found),
|
||||
"bytes": sum(u.size for u in found),
|
||||
"archives": sum(1 for u in found if u.task_type == "archive"),
|
||||
"artists": artists,
|
||||
**passed,
|
||||
}
|
||||
return found, summary
|
||||
|
||||
|
||||
def queue_unimported(
|
||||
session: Session, images_root: Path, *, triggered_by: str = "manual",
|
||||
) -> dict:
|
||||
"""Create a `library` import batch holding every file `find_unimported`
|
||||
lists, and queue it. Returns the same summary plus `batch_id` (None when
|
||||
there was nothing to import)."""
|
||||
from ..tasks.import_file import enqueue_import # tasks import services
|
||||
|
||||
found, summary = find_unimported(session, images_root)
|
||||
summary["batch_id"] = None
|
||||
if not found:
|
||||
return summary
|
||||
batch = ImportBatch(
|
||||
triggered_by=triggered_by, source_path=str(images_root),
|
||||
scan_mode="library", status="running", total_files=len(found),
|
||||
)
|
||||
session.add(batch)
|
||||
session.flush()
|
||||
tasks = [
|
||||
ImportTask(
|
||||
batch_id=batch.id, source_path=str(u.path), task_type=u.task_type,
|
||||
status="queued", size_bytes=u.size,
|
||||
)
|
||||
for u in found
|
||||
]
|
||||
session.add_all(tasks)
|
||||
session.commit()
|
||||
for t in tasks:
|
||||
enqueue_import(t.id, t.task_type)
|
||||
summary["batch_id"] = batch.id
|
||||
return summary
|
||||
|
||||
|
||||
def folder_owner(
|
||||
session: Session, path: Path, images_root: Path,
|
||||
) -> tuple[Artist | None, Source | None]:
|
||||
"""The artist whose folder holds `path`, and that artist's source for the
|
||||
platform folder below it when there is exactly one (a Discord artist has a
|
||||
source per channel, so there the post's sidecar decides)."""
|
||||
parts = Path(path).relative_to(images_root).parts
|
||||
artist = session.execute(
|
||||
select(Artist).where(Artist.slug == slugify(parts[0]))
|
||||
).scalar_one_or_none()
|
||||
if artist is None or len(parts) < 3:
|
||||
return artist, None
|
||||
sources = session.execute(
|
||||
select(Source).where(Source.artist_id == artist.id, Source.platform == parts[1])
|
||||
).scalars().all()
|
||||
return artist, (sources[0] if len(sources) == 1 else None)
|
||||
@@ -97,8 +97,8 @@ def _sigmoid(z, np):
|
||||
|
||||
def _conflict_scores(Xn, Wc, bc, np):
|
||||
"""The presentation conflict signal (#141): per row, the MAX content-head
|
||||
probability and WHICH head produced it. Shared by the system-tag sweep's guard-2
|
||||
and the soft-wip audit — both ask "does this ALSO look like real content?"."""
|
||||
probability and WHICH head produced it — the system-tag sweep's guard-2 asks
|
||||
"does this ALSO look like real content?"."""
|
||||
cprobs = _sigmoid(Xn @ Wc.T + bc, np)
|
||||
return cprobs.max(axis=1), cprobs.argmax(axis=1)
|
||||
|
||||
@@ -106,10 +106,9 @@ def _conflict_scores(Xn, Wc, bc, np):
|
||||
def _insert_presentation_review(
|
||||
session, *, image_record_id, tag_id, conflict_tag_id, conflict_score, mode,
|
||||
):
|
||||
"""Single-source the ring-loud PresentationReview row shape so the two writers
|
||||
(system-tag sweep guard-2 + soft-wip audit) can't drift on columns or `mode` —
|
||||
they share the (image_record_id, tag_id) composite PK, so a divergent `mode`
|
||||
would be a silent first-writer-wins bug."""
|
||||
"""Single-source the ring-loud PresentationReview row shape, so every writer of
|
||||
the (image_record_id, tag_id) composite PK agrees on columns and `mode` — a
|
||||
divergent `mode` would be a silent first-writer-wins bug."""
|
||||
session.execute(
|
||||
pg_insert(PresentationReview)
|
||||
.values(
|
||||
@@ -963,68 +962,6 @@ def system_tag_auto_apply_sweep(
|
||||
}
|
||||
|
||||
|
||||
def soft_wip_conflict_audit(session: Session, dry_run: bool = False) -> dict:
|
||||
"""Ring-loud audit for the SOFT WIP-title cohort (#1474). Images auto-tagged
|
||||
`wip` from a low-precision sketch/doodle title (source='wip_title_soft') that ALSO
|
||||
score >= the process conflict threshold on a content head are probably FINISHED
|
||||
art mis-tagged as process — flag them (PresentationReview, mode='process') so the
|
||||
review strip surfaces them ("also looks like <X>", Keep tag / Remove tag). Does
|
||||
NOT remove the tag; the operator decides. No-op when there are no content heads.
|
||||
numpy-only. Returns {n_scanned, n_flagged}."""
|
||||
import numpy as np
|
||||
|
||||
from ..wip_title import WIP_TITLE_SOFT_SOURCE, resolve_wip_tag_id
|
||||
|
||||
settings = _settings(session)
|
||||
ver = settings.embedder_model_version
|
||||
conflict_thr = float(settings.process_conflict_threshold)
|
||||
conf = _conflict_heads(session, ver)
|
||||
wip_id = resolve_wip_tag_id(session)
|
||||
if not conf or wip_id is None:
|
||||
return {"n_scanned": 0, "n_flagged": 0}
|
||||
Wc = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in conf])
|
||||
bc = np.asarray([r.bias for r in conf], dtype=np.float32)
|
||||
conf_tag_ids = [r.tag_id for r in conf]
|
||||
|
||||
soft_ids = [iid for (iid,) in session.execute(
|
||||
select(image_tag.c.image_record_id)
|
||||
.where(image_tag.c.tag_id == wip_id)
|
||||
.where(image_tag.c.source == WIP_TITLE_SOFT_SOURCE)
|
||||
)]
|
||||
# Skip images already flagged for this tag (idempotent re-runs).
|
||||
flagged = {iid for (iid,) in session.execute(
|
||||
select(PresentationReview.image_record_id)
|
||||
.where(PresentationReview.tag_id == wip_id)
|
||||
)}
|
||||
soft_ids = [i for i in soft_ids if i not in flagged]
|
||||
|
||||
n_flagged = 0
|
||||
scanned = 0
|
||||
for start in range(0, len(soft_ids), _AUTO_APPLY_CHUNK):
|
||||
chunk = soft_ids[start:start + _AUTO_APPLY_CHUNK]
|
||||
emb = _load_embeddings(session, chunk)
|
||||
cids = [i for i in chunk if i in emb]
|
||||
if not cids:
|
||||
continue
|
||||
scanned += len(cids)
|
||||
Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np)
|
||||
max_c, arg_c = _conflict_scores(Xn, Wc, bc, np)
|
||||
for k in range(len(cids)):
|
||||
if float(max_c[k]) >= conflict_thr:
|
||||
n_flagged += 1
|
||||
if not dry_run:
|
||||
_insert_presentation_review(
|
||||
session,
|
||||
image_record_id=cids[k], tag_id=wip_id,
|
||||
conflict_tag_id=conf_tag_ids[int(arg_c[k])],
|
||||
conflict_score=float(max_c[k]),
|
||||
mode="process",
|
||||
)
|
||||
if not dry_run:
|
||||
session.commit()
|
||||
return {"n_scanned": scanned, "n_flagged": n_flagged}
|
||||
|
||||
|
||||
def retract_auto_applied_heads(session: Session) -> int:
|
||||
"""Soft auto-apply (milestone 139): re-score every standing source='head_auto'
|
||||
tag against its CURRENT head and REMOVE the ones now BELOW the head's
|
||||
|
||||
@@ -32,11 +32,8 @@ from ...models.tag import image_tag
|
||||
# `process_auto` (#1464): wip/editor screenshot applied by the process sweep are
|
||||
# ALSO provisional — the head must learn only from title (`wip_title`) + manual
|
||||
# labels, never its own auto-applied output, or it would runaway (operator 2026-07-12).
|
||||
# `wip_title_soft` (#1474): the soft title tier (sketch/doodle) is LOW-precision, so
|
||||
# it's provisional too — a finished piece titled "sketch" must not train the wip head.
|
||||
_AUTO_SOURCES = (
|
||||
"head_auto", "ccip_auto", "ml_auto", "presentation_auto", "process_auto",
|
||||
"wip_title_soft",
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
@@ -184,6 +185,56 @@ def post_dir_name(post: dict) -> str:
|
||||
return sanitize_segment(raw)
|
||||
|
||||
|
||||
# -- one downloadable file, and its seen-ledger identity -------------------
|
||||
|
||||
@dataclass
|
||||
class MediaItem:
|
||||
"""One resolved downloadable file belonging to a post, as every platform
|
||||
client returns it.
|
||||
|
||||
- `url`: what to fetch. `filename`: the media's own name when the platform
|
||||
gives one, else the URL basename (never a network call).
|
||||
- `kind`: the content type, using gallery-dl's `files` names ("images",
|
||||
"attachments", …) so per-source content_types filter the same way.
|
||||
- `post_id`: the owning post, so a flattened list stays traceable.
|
||||
- `filehash`: the 32-hex content hash in a content-addressed CDN URL
|
||||
(Patreon), else None.
|
||||
- `media_id`: the platform's stable per-file id where the URL has no hash:
|
||||
SubscribeStar's gallery id, Discord's attachment id (or, for an embed, a
|
||||
hash of its URL path; the query string is a signature that changes on
|
||||
every fetch). Empty when the platform has none.
|
||||
- `extension` / `num`: Discord only. gallery-dl splits the name from its
|
||||
extension and numbers files 1-based across the message; both name the
|
||||
file on disk.
|
||||
"""
|
||||
|
||||
url: str
|
||||
filename: str
|
||||
kind: str
|
||||
post_id: str
|
||||
filehash: str | None = None
|
||||
media_id: str = ""
|
||||
extension: str = ""
|
||||
num: int = 0
|
||||
|
||||
|
||||
# The seen-ledger tables' key column width.
|
||||
_LEDGER_KEY_MAX = 128
|
||||
|
||||
|
||||
def ledger_key(media: MediaItem) -> str:
|
||||
"""A file's identity in the cross-run seen-ledger, stable across walks.
|
||||
|
||||
The content hash when the URL carries one. Otherwise the file is keyed
|
||||
within its post: by the platform's per-file id, or by filename when there is
|
||||
none (a Patreon Mux video or a hashless inline image). A positional key is
|
||||
never used, because an edit that removes a file renumbers the ones after it
|
||||
and would make a different file read as seen."""
|
||||
if media.filehash:
|
||||
return media.filehash
|
||||
return f"{media.post_id}:{media.media_id or media.filename}"[:_LEDGER_KEY_MAX]
|
||||
|
||||
|
||||
# -- per-item download outcomes (shared dataclasses) -----------------------
|
||||
|
||||
@dataclass
|
||||
@@ -267,11 +318,16 @@ class Membership:
|
||||
# -- base downloader (shared fetch/validate plumbing) ----------------------
|
||||
|
||||
class BaseNativeDownloader:
|
||||
"""Shared download plumbing for native-platform downloaders: the streaming
|
||||
GET (transient-retry + Range-resume) and file validation/quarantine. Platform
|
||||
downloaders subclass this and implement `download_post` / `write_post_record`
|
||||
/ the per-media sidecar (and any platform-specific fetch, e.g. Patreon's
|
||||
Mux/yt-dlp video branch). PURE: no DB; the seen-skip is an injected predicate.
|
||||
"""Shared download plumbing for native-platform downloaders: one post's
|
||||
files (`download_post`, with the seen/disk skips and the time-box), the
|
||||
streaming GET (transient-retry + Range-resume), and file
|
||||
validation/quarantine.
|
||||
|
||||
A platform downloader subclasses this, implements `write_post_record`, and
|
||||
overrides only the seams where its layout differs: `_post_folder`,
|
||||
`_media_name`, `_media_sidecar`, and for Patreon's Mux/yt-dlp video,
|
||||
`_existing_output` and `_fetch_media`. PURE: no DB; the seen-skip is an
|
||||
injected predicate.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -383,18 +439,118 @@ class BaseNativeDownloader:
|
||||
)
|
||||
return reason, (dest or path)
|
||||
|
||||
# -- sidecar (per-media, minimal) --------------------------------------
|
||||
# -- one post's files ---------------------------------------------------
|
||||
|
||||
def _write_minimal_sidecar(
|
||||
self, post: dict, media_path: Path, *, source_url: str | None = None
|
||||
) -> Path:
|
||||
"""Post-first per-media sidecar (#856): image identity ONLY
|
||||
(category/id/source_url). The post body/links live solely in _post.json."""
|
||||
data: dict = {"category": self.platform, "id": str(post.get("id") or "")}
|
||||
if source_url:
|
||||
data["source_url"] = source_url
|
||||
def download_post(
|
||||
self,
|
||||
post: dict,
|
||||
media_items: list[MediaItem],
|
||||
artist_slug: str,
|
||||
*,
|
||||
is_seen: Callable[[MediaItem], bool] = lambda m: False,
|
||||
should_stop: Callable[[], bool] = lambda: False,
|
||||
recapture: bool = False,
|
||||
) -> list[MediaOutcome]:
|
||||
"""Download every file of one post and return one outcome per file.
|
||||
|
||||
- Two-tier skip: the injected seen-ledger predicate first, then the disk.
|
||||
- `should_stop()` is polled before each file, so a file-dense post
|
||||
cannot run a time-boxed chunk far past its deadline. The files not
|
||||
reached are not marked seen and are fetched by the next chunk.
|
||||
- One file's failure is an "error" outcome for that file; the rest
|
||||
proceed.
|
||||
- `recapture` re-reads what is on disk without re-downloading: a seen
|
||||
file that is on disk comes back `skipped_disk` with its path (so the
|
||||
engine can backfill its source filehash), and a seen file that is gone
|
||||
is left alone, since refetching is recovery's job.
|
||||
"""
|
||||
folder = self._post_folder(post, artist_slug)
|
||||
outcomes: list[MediaOutcome] = []
|
||||
for index, media in enumerate(media_items, start=1):
|
||||
if should_stop():
|
||||
break
|
||||
try:
|
||||
outcomes.append(self._download_one(
|
||||
post, media, folder, artist_slug, index, is_seen,
|
||||
recapture=recapture,
|
||||
))
|
||||
except Exception as exc: # resilient: isolate one item's failure
|
||||
log.warning(
|
||||
"%s media failed (post %s, item %d): %s",
|
||||
self.platform, post.get("id"), index, exc,
|
||||
)
|
||||
outcomes.append(
|
||||
MediaOutcome(media=media, status="error", path=None, error=str(exc))
|
||||
)
|
||||
return outcomes
|
||||
|
||||
def _download_one(
|
||||
self,
|
||||
post: dict,
|
||||
media: MediaItem,
|
||||
folder: Path,
|
||||
artist_slug: str,
|
||||
index: int,
|
||||
is_seen: Callable[[MediaItem], bool],
|
||||
*,
|
||||
recapture: bool = False,
|
||||
) -> MediaOutcome:
|
||||
seen = is_seen(media)
|
||||
if seen and not recapture:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
path = folder / self._media_name(post, media, index)
|
||||
existing = self._existing_output(path, media)
|
||||
if existing is not None:
|
||||
return MediaOutcome(media=media, status="skipped_disk", path=existing, error=None)
|
||||
if seen:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
# Real downloads are paced; the skips above have already returned.
|
||||
if self._rate_limit > 0:
|
||||
time.sleep(self._rate_limit)
|
||||
return self._fetch_media(post, media, path, artist_slug)
|
||||
|
||||
# -- platform seams ----------------------------------------------------
|
||||
# Where a platform lays its files out. Each matches the layout gallery-dl
|
||||
# wrote, so files from before a platform moved to its native downloader
|
||||
# are found on disk instead of fetched again.
|
||||
|
||||
def _post_folder(self, post: dict, artist_slug: str) -> Path:
|
||||
return self.images_root / artist_slug / self.platform / post_dir_name(post)
|
||||
|
||||
def _media_name(self, post: dict, media: MediaItem, index: int) -> str:
|
||||
return sanitize_segment(f"{index:02d}_{media.filename}")
|
||||
|
||||
def _existing_output(self, path: Path, media: MediaItem) -> Path | None:
|
||||
"""The file already on disk for this media, or None."""
|
||||
return path if path.exists() else None
|
||||
|
||||
def _fetch_media(
|
||||
self, post: dict, media: MediaItem, path: Path, artist_slug: str,
|
||||
) -> MediaOutcome:
|
||||
"""Fetch one file, validate it, and write its sidecar."""
|
||||
out = self._fetch_get(media.url, path)
|
||||
reason, quarantined = self._validate_path(out, artist_slug, media.url)
|
||||
if reason is not None:
|
||||
# Distinct from an error, so the run reports a real
|
||||
# files_quarantined count and the paths.
|
||||
return MediaOutcome(media=media, status="quarantined", path=quarantined, error=reason)
|
||||
self._write_media_sidecar(post, out, media)
|
||||
return MediaOutcome(media=media, status="downloaded", path=out, error=None)
|
||||
|
||||
def _media_sidecar(self, post: dict, media: MediaItem) -> dict:
|
||||
"""The per-file sidecar's content: the file's identity only. The post's
|
||||
body, title and links live in its post record alone, so they are never
|
||||
copied beside every file."""
|
||||
return {"category": self.platform, "id": str(post.get("id") or ""),
|
||||
"source_url": media.url}
|
||||
|
||||
def _write_media_sidecar(self, post: dict, media_path: Path, media: MediaItem) -> Path:
|
||||
"""Write the per-file sidecar beside the file, named like it minus the
|
||||
extension, which is what `find_sidecar` pairs first."""
|
||||
sidecar_path = media_path.with_suffix(".json")
|
||||
sidecar_path.write_text(json.dumps(data, indent=2))
|
||||
sidecar_path.write_text(json.dumps(self._media_sidecar(post, media), indent=2))
|
||||
return sidecar_path
|
||||
|
||||
# --- membership status vocabulary (#387) ------------------------------------
|
||||
|
||||
@@ -38,14 +38,15 @@ FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from html import unescape
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
from urllib.parse import parse_qs, unquote, urlsplit
|
||||
|
||||
import requests
|
||||
|
||||
@@ -53,6 +54,7 @@ from ..utils.paths import filehash_from_url
|
||||
from ..utils.prosemirror import post_body_html
|
||||
from .native_ingest_common import (
|
||||
_MAX_429_RETRIES,
|
||||
MediaItem,
|
||||
Membership,
|
||||
NativeAuthError,
|
||||
NativeDriftError,
|
||||
@@ -140,29 +142,25 @@ class PatreonDriftError(PatreonAPIError, NativeDriftError):
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class MediaItem:
|
||||
"""One resolved downloadable item belonging to a post.
|
||||
# Patreon's CDN URLs carry a base64 JSON segment of render options, such as
|
||||
# eyJ3Ijo2MjB9 for {"w":620}. A locked preview is rendered blurred and says so
|
||||
# with a "b" (blur) option. A post the account can view can still embed media
|
||||
# from a post it can't, such as a public teaser showing a paid post's image,
|
||||
# and that media arrives blurred even though the post itself isn't gated.
|
||||
_RENDER_OPTIONS_RE = re.compile(r"/(eyJ[A-Za-z0-9_\-]+(?:=|%3D)*)(?:/|$)")
|
||||
|
||||
Fields:
|
||||
url — the CDN/download URL to fetch.
|
||||
filename — media `file_name` when present; otherwise the URL basename
|
||||
(NEVER a network call). Bounded/sane extension.
|
||||
kind — one of: "images", "image_large", "attachments", "postfile",
|
||||
"content". Mirrors gallery-dl's `files` content-type names so
|
||||
the later step can honor the same per-source content_types.
|
||||
filehash — the 32-char hex (MD5) segment from the CDN URL, or None if
|
||||
the URL carries no such segment. Used for in-post dedup and
|
||||
the cross-run seen-ledger.
|
||||
post_id — the owning post's id (so a flattened media list stays
|
||||
traceable to its post).
|
||||
"""
|
||||
|
||||
url: str
|
||||
filename: str
|
||||
kind: str
|
||||
filehash: str | None
|
||||
post_id: str
|
||||
def is_blurred_preview(url: str) -> bool:
|
||||
"""True when a Patreon CDN URL is a blurred locked preview."""
|
||||
m = _RENDER_OPTIONS_RE.search(urlsplit(url).path)
|
||||
if not m:
|
||||
return False
|
||||
token = unquote(m.group(1))
|
||||
try:
|
||||
options = json.loads(base64.urlsafe_b64decode(token + "=" * (-len(token) % 4)))
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
return isinstance(options, dict) and bool(options.get("b"))
|
||||
|
||||
|
||||
def _filehash(url: str) -> str | None:
|
||||
@@ -479,6 +477,11 @@ class PatreonClient:
|
||||
)
|
||||
)
|
||||
|
||||
blurred = [i for i in items if is_blurred_preview(i.url)]
|
||||
if blurred:
|
||||
log.info("Patreon post %s: skipped %d blurred locked preview(s)",
|
||||
post_id, len(blurred))
|
||||
items = [i for i in items if not is_blurred_preview(i.url)]
|
||||
return _dedup_by_filehash(items)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -43,10 +43,10 @@ from .native_ingest_common import (
|
||||
_BACKOFF_CAP_SECONDS,
|
||||
_MAX_MEDIA_RETRIES,
|
||||
BaseNativeDownloader,
|
||||
MediaItem,
|
||||
MediaOutcome,
|
||||
PostRecordOutcome,
|
||||
post_dir_name,
|
||||
sanitize_segment,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -102,139 +102,33 @@ class PatreonDownloader(BaseNativeDownloader):
|
||||
# None in unit tests / when enrichment isn't wanted.
|
||||
self._content_fetcher = content_fetcher
|
||||
|
||||
# -- public ------------------------------------------------------------
|
||||
# The download loop, layout and per-file sidecar are BaseNativeDownloader's.
|
||||
# Patreon adds one branch: a Mux/HLS video is fetched with yt-dlp, which may
|
||||
# remux it to a container of its own choosing.
|
||||
|
||||
def download_post(
|
||||
self,
|
||||
post: dict,
|
||||
media_items: list,
|
||||
artist_slug: str,
|
||||
*,
|
||||
is_seen: Callable[[object], bool] = lambda m: False,
|
||||
should_stop: Callable[[], bool] = lambda: False,
|
||||
recapture: bool = False,
|
||||
) -> list[MediaOutcome]:
|
||||
"""Download every media item of one post; return per-item outcomes.
|
||||
def _existing_output(self, path: Path, media: MediaItem) -> Path | None:
|
||||
if path.exists():
|
||||
return path
|
||||
if _is_video_url(media.url):
|
||||
# A remux lands at a different extension; honour one already there.
|
||||
return self._existing_video_output(path)
|
||||
return None
|
||||
|
||||
Builds the post directory, iterates media (1-based NN), applies the
|
||||
two-tier skip (injected is_seen, then disk), downloads (plain GET or
|
||||
yt-dlp for video), writes the sidecar for each freshly-downloaded item,
|
||||
validates, and returns outcomes. Resilient: one media's failure yields
|
||||
an "error" outcome for that item; the rest proceed.
|
||||
|
||||
`should_stop()` is polled BEFORE each media item: a media-dense post can
|
||||
otherwise run a backfill chunk far past its time-box (the engine only
|
||||
re-checks the budget between posts), so we honour the deadline mid-post
|
||||
and return the items done so far — the rest re-fetch next chunk (they
|
||||
were never marked seen). Bounds chunk overrun to one media download.
|
||||
|
||||
`recapture` (#830): don't re-download already-present media, but DO surface
|
||||
on-disk media as `skipped_disk` (with its path) even when the seen-ledger
|
||||
would tier-1 skip it — so the engine can backfill source_filehash for
|
||||
inline-image localization. Genuinely-missing seen media is NOT refetched.
|
||||
"""
|
||||
post_dir = self.images_root / artist_slug / "patreon" / post_dir_name(post)
|
||||
outcomes: list[MediaOutcome] = []
|
||||
|
||||
for i, media in enumerate(media_items, start=1):
|
||||
if should_stop():
|
||||
break
|
||||
try:
|
||||
outcomes.append(
|
||||
self._download_one(
|
||||
post, media, post_dir, artist_slug, i, is_seen,
|
||||
recapture=recapture,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # resilient: isolate one item's failure
|
||||
log.warning(
|
||||
"Patreon media failed (post %s, item %d): %s",
|
||||
post.get("id"), i, exc,
|
||||
)
|
||||
outcomes.append(
|
||||
MediaOutcome(media=media, status="error", path=None, error=str(exc))
|
||||
)
|
||||
return outcomes
|
||||
|
||||
# -- per-item ----------------------------------------------------------
|
||||
|
||||
def _download_one(
|
||||
self,
|
||||
post: dict,
|
||||
media,
|
||||
post_dir: Path,
|
||||
artist_slug: str,
|
||||
index: int,
|
||||
is_seen: Callable[[object], bool],
|
||||
*,
|
||||
recapture: bool = False,
|
||||
def _fetch_media(
|
||||
self, post: dict, media: MediaItem, path: Path, artist_slug: str,
|
||||
) -> MediaOutcome:
|
||||
# tier-1: seen ledger (injected; no DB here). In recapture mode we DON'T
|
||||
# short-circuit here — we fall through to the disk check so an on-disk
|
||||
# seen file is surfaced as skipped_disk (with its path) for source_filehash
|
||||
# backfill; a seen file that's NOT on disk is left alone (not refetched).
|
||||
seen = is_seen(media)
|
||||
if seen and not recapture:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
nn = f"{index:02d}"
|
||||
final_name = sanitize_segment(f"{nn}_{media.filename}")
|
||||
media_path = post_dir / final_name
|
||||
|
||||
# tier-2: already on disk.
|
||||
if media_path.exists():
|
||||
if not _is_video_url(media.url):
|
||||
return super()._fetch_media(post, media, path, artist_slug)
|
||||
out_path = self._run_ytdlp(media.url, path, _VIDEO_HEADERS)
|
||||
if out_path is None or not Path(out_path).exists():
|
||||
return MediaOutcome(
|
||||
media=media, status="skipped_disk", path=media_path, error=None
|
||||
media=media, status="error", path=None, error="yt-dlp produced no output",
|
||||
)
|
||||
# Video may land at a different extension; honor a pre-existing remux.
|
||||
if _is_video_url(media.url):
|
||||
existing = self._existing_video_output(media_path)
|
||||
if existing is not None:
|
||||
return MediaOutcome(
|
||||
media=media, status="skipped_disk", path=existing, error=None
|
||||
)
|
||||
|
||||
# recapture: a seen item that isn't on disk is NOT re-downloaded (that's
|
||||
# recovery's job) — recapture only re-grabs post text + localizes existing
|
||||
# files. Returns skipped_seen so the run-of-seen / counts stay consistent.
|
||||
if seen:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
post_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Pace real downloads only (the skips above already returned). plan #703.
|
||||
if self._rate_limit > 0:
|
||||
time.sleep(self._rate_limit)
|
||||
|
||||
if _is_video_url(media.url):
|
||||
out_path = self._run_ytdlp(media.url, media_path, _VIDEO_HEADERS)
|
||||
if out_path is None or not Path(out_path).exists():
|
||||
return MediaOutcome(
|
||||
media=media,
|
||||
status="error",
|
||||
path=None,
|
||||
error="yt-dlp produced no output",
|
||||
)
|
||||
out_path = Path(out_path)
|
||||
else:
|
||||
out_path = self._fetch_get(media.url, media_path)
|
||||
reason, quarantine_dest = self._validate_path(
|
||||
out_path, artist_slug, media.url
|
||||
)
|
||||
if reason is not None:
|
||||
# Quarantined (corrupt/invalid) — distinct from a download error
|
||||
# so the run can report a real files_quarantined count + paths.
|
||||
return MediaOutcome(
|
||||
media=media, status="quarantined",
|
||||
path=quarantine_dest, error=reason,
|
||||
)
|
||||
|
||||
self._write_sidecar(post, out_path, source_url=media.url)
|
||||
out_path = Path(out_path)
|
||||
self._write_media_sidecar(post, out_path, media)
|
||||
return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
|
||||
|
||||
# -- video (Mux/HLS via yt-dlp) ----------------------------------------
|
||||
# The plain-GET streaming path (_fetch_get / _fetch_to_file) and
|
||||
# _validate_path are inherited from BaseNativeDownloader.
|
||||
|
||||
def _run_ytdlp(self, url: str, dest: Path, headers: dict) -> Path | None:
|
||||
"""Invoke yt-dlp to fetch a Mux/HLS stream to (around) `dest`.
|
||||
@@ -316,43 +210,12 @@ class PatreonDownloader(BaseNativeDownloader):
|
||||
return cand
|
||||
return None
|
||||
|
||||
# -- sidecar -----------------------------------------------------------
|
||||
|
||||
def _write_sidecar(
|
||||
self, post: dict, media_path: Path, *, source_url: str | None = None
|
||||
def _write_post_record_file(
|
||||
self, post: dict, sidecar_path: Path, *, detail_fetch: bool = True,
|
||||
) -> Path:
|
||||
"""Write the per-media sidecar next to `media_path` — post-first (#856).
|
||||
|
||||
On the native ingester the POST-RECORD (`write_post_record` → `_post.json`)
|
||||
is the sole writer of the post body/links/metadata, captured once per post
|
||||
BEFORE its media in the walk. So the per-media sidecar carries ONLY
|
||||
image-specific identity: `category` (platform) + `id` (external_post_id, to
|
||||
link provenance to the right Post) + this file's `source_url` (its CDN URL,
|
||||
#830 Phase 2 — the importer persists its filehash so the body's inline
|
||||
`<img src>` remaps to the local copy at render time). No body: writing it
|
||||
next to every image duplicated the post body N+1× and risked divergence
|
||||
(milestone #67). The importer skips post fields for these (post_first).
|
||||
"""
|
||||
return self._write_sidecar_data(
|
||||
post, media_path.with_suffix(".json"), source_url=source_url,
|
||||
minimal=True,
|
||||
)
|
||||
|
||||
def _write_sidecar_data(
|
||||
self, post: dict, sidecar_path: Path, *, source_url: str | None = None,
|
||||
minimal: bool = False, detail_fetch: bool = True,
|
||||
) -> Path:
|
||||
"""Serialize the post's metadata to `sidecar_path`. The post-only record
|
||||
(`write_post_record`) writes the FULL post (body/title/date/url); the
|
||||
per-media sidecar (`_write_sidecar`, minimal=True) writes only image
|
||||
identity (category/id/source_url) — post-first (#856). `source_url` is set
|
||||
only for the per-media sidecar — a media-less post has no source file."""
|
||||
if minimal:
|
||||
data = {"category": "patreon", "id": str(post.get("id") or "")}
|
||||
if source_url:
|
||||
data["source_url"] = source_url
|
||||
sidecar_path.write_text(json.dumps(data, indent=2))
|
||||
return sidecar_path
|
||||
"""Serialize the full post record (body, title, date, url) to
|
||||
`sidecar_path`. The per-file sidecars carry only the file's identity
|
||||
(BaseNativeDownloader._media_sidecar)."""
|
||||
attrs = post.get("attributes") or {}
|
||||
title = attrs.get("title")
|
||||
# Resolve the body HTML from the feed attrs: legacy flat `content`, else
|
||||
@@ -387,8 +250,6 @@ class PatreonDownloader(BaseNativeDownloader):
|
||||
"published_at": published if isinstance(published, str) else None,
|
||||
"url": url if isinstance(url, str) else None,
|
||||
}
|
||||
if source_url:
|
||||
data["source_url"] = source_url
|
||||
sidecar_path.write_text(json.dumps(data, indent=2))
|
||||
return sidecar_path
|
||||
|
||||
@@ -434,10 +295,10 @@ class PatreonDownloader(BaseNativeDownloader):
|
||||
)
|
||||
post_dir = self.images_root / artist_slug / "patreon" / post_dir_name(post)
|
||||
post_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = self._write_sidecar_data(
|
||||
path = self._write_post_record_file(
|
||||
post, post_dir / "_post.json", detail_fetch=not revisit,
|
||||
)
|
||||
# _write_sidecar_data has by now memoized any detail-fetched body onto
|
||||
# _write_post_record_file has by now memoized any detail-fetched body onto
|
||||
# post["attributes"]["content"], so re-read it for the FINAL char count.
|
||||
body = (post.get("attributes") or {}).get("content")
|
||||
body_chars = len(body) if isinstance(body, str) else 0
|
||||
|
||||
@@ -6,9 +6,8 @@ media → tiered skip → download → mark-seen / record-failures / checkpoint-
|
||||
now lives in `ingest_core.Ingester` — it's identical for every platform. This
|
||||
module is the thin Patreon adapter: it wires the Patreon `client`/`downloader`/
|
||||
ledger models/constraints/key into the core and supplies the Patreon-specific
|
||||
failure mapping. `download_service.download_source` calls `PatreonIngester.run`
|
||||
exactly as before; the public surface (this class, `_ledger_key`,
|
||||
`DEAD_LETTER_THRESHOLD`, `verify_patreon_credential`) is unchanged.
|
||||
failure mapping. `download_service.download_source` calls `PatreonIngester.run`;
|
||||
the seen-ledger key is the shared `native_ingest_common.ledger_key`.
|
||||
|
||||
Three modes (selected by `download_service` from `config_overrides` state):
|
||||
- tick — newest→oldest, skip seen (tier-1 ledger + tier-2 disk), early-out
|
||||
@@ -36,38 +35,19 @@ from pathlib import Path
|
||||
|
||||
from ..models import PatreonFailedMedia, PatreonSeenMedia
|
||||
from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester
|
||||
from .patreon_client import MediaItem, PatreonAPIError, PatreonClient
|
||||
from .native_ingest_common import ledger_key
|
||||
from .patreon_client import PatreonAPIError, PatreonClient
|
||||
from .patreon_downloader import PatreonDownloader
|
||||
from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
|
||||
|
||||
__all__ = [
|
||||
"DEAD_LETTER_THRESHOLD",
|
||||
"PatreonIngester",
|
||||
"_ledger_key",
|
||||
"verify_patreon_credential",
|
||||
]
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Ledger keys are stored in patreon_seen_media.filehash VARCHAR(128); bound any
|
||||
# synthesized key so a pathologically long file_name can't overflow the column.
|
||||
_LEDGER_KEY_MAX = 128
|
||||
|
||||
|
||||
def _ledger_key(media: MediaItem) -> str:
|
||||
"""Stable per-media identity for the cross-run seen-ledger.
|
||||
|
||||
A Patreon CDN URL carries a 32-char MD5 (`media.filehash`) — that is the
|
||||
natural key. Some media have none: Mux/HLS video (`stream.mux.com`, no
|
||||
content hash at discovery) and the odd inline-content `<img>` pointing at a
|
||||
hashless URL. The plan calls the video case the ``video:<post_id>:<media_id>``
|
||||
sentinel; `MediaItem` carries no media_id, so the post-scoped filename is the
|
||||
stable proxy. Bounded to the column width.
|
||||
"""
|
||||
if media.filehash:
|
||||
return media.filehash
|
||||
return f"{media.post_id}:{media.filename}"[:_LEDGER_KEY_MAX]
|
||||
|
||||
|
||||
class PatreonIngester(Ingester):
|
||||
"""Walk a Patreon campaign's posts, download unseen media, return a
|
||||
@@ -124,7 +104,7 @@ class PatreonIngester(Ingester):
|
||||
failed_model=PatreonFailedMedia,
|
||||
seen_constraint="uq_patreon_seen_media_source_id",
|
||||
failed_constraint="uq_patreon_failed_media_source_id",
|
||||
ledger_key=_ledger_key,
|
||||
ledger_key=ledger_key,
|
||||
platform="patreon",
|
||||
error_base=PatreonAPIError,
|
||||
# API_DRIFT message phrasing; the base Ingester._failure_result owns
|
||||
|
||||
@@ -17,16 +17,16 @@ import logging
|
||||
import redis
|
||||
|
||||
from ..config import get_config
|
||||
from .download_backends import NATIVE_INGESTER_PLATFORMS
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Platforms walked one-at-a-time. gallery-dl platforms are intentionally NOT
|
||||
# here: each runs as a self-pacing subprocess and they're lower-volume. The
|
||||
# native-ingester platforms are serialized (one paced scrape/API walk at a time).
|
||||
# Add a platform here to cap it to a single concurrent walk. Discord most of
|
||||
# all: every source walks on the operator's ONE user token, and parallel walks
|
||||
# Platforms walked one-at-a-time: every native-ingester platform, one paced
|
||||
# scrape/API walk at a time. gallery-dl platforms are not: each runs as a
|
||||
# self-pacing subprocess and they are lower-volume. Discord most of all needs
|
||||
# it: every source walks on the operator's ONE user token, and parallel walks
|
||||
# on a user account are both how its rate limit trips and what gets it flagged.
|
||||
SERIALIZED_PLATFORMS = frozenset({"patreon", "subscribestar", "discord"})
|
||||
SERIALIZED_PLATFORMS = NATIVE_INGESTER_PLATFORMS
|
||||
|
||||
_LOCK_PREFIX = "fc:download_lock:"
|
||||
|
||||
@@ -60,3 +60,22 @@ def platform_lock(platform: str, *, ttl_seconds: int):
|
||||
except redis.RedisError as exc: # pragma: no cover - broker outage
|
||||
log.warning("platform_lock unavailable for %s: %s", platform, exc)
|
||||
return None
|
||||
|
||||
|
||||
def release_all_platform_locks() -> int:
|
||||
"""Drop every serialized platform's lock. Returns how many were held.
|
||||
|
||||
Only for the download lane's boot (#4433). A worker that restarts mid-walk
|
||||
is SIGKILLed past its stop grace, so its `finally` never releases the lock,
|
||||
and the TTL keeps every other source on that platform bouncing for up to
|
||||
27 minutes after the new worker is ready. At boot no walk of ours can be
|
||||
running, so a held lock names a dead one. Assumes one download consumer —
|
||||
the only shape FC deploys; a second replica booting would free a live
|
||||
walk's lock (not corrupt it: the walk runs on, a second walk may overlap it).
|
||||
"""
|
||||
try:
|
||||
client = _redis()
|
||||
return int(client.delete(*(f"{_LOCK_PREFIX}{p}" for p in SERIALIZED_PLATFORMS)))
|
||||
except redis.RedisError as exc: # pragma: no cover - broker outage
|
||||
log.warning("could not release platform locks at boot: %s", exc)
|
||||
return 0
|
||||
|
||||
@@ -479,6 +479,10 @@ class PostFeedService:
|
||||
# the UI uses this to explain why a post it linked to is not in the
|
||||
# stream.
|
||||
"absorbed_by_post_id": post.absorbed_by_post_id,
|
||||
# #4481. The Discord channel a message was posted in: both the native
|
||||
# post record and gallery-dl's sidecars carry it as `channel`. None
|
||||
# on every other platform, and on a Discord post that never said.
|
||||
"channel": _discord_channel(post, source),
|
||||
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
||||
"source": (
|
||||
{"id": source.id, "platform": source.platform}
|
||||
@@ -488,3 +492,13 @@ class PostFeedService:
|
||||
"thumbnails_more": thumbs_entry["more"],
|
||||
"attachments": atts_map.get(post.id, []),
|
||||
}
|
||||
|
||||
|
||||
def _discord_channel(post: Post, source: Source | None) -> str | None:
|
||||
if source is None or source.platform != "discord":
|
||||
return None
|
||||
raw = post.raw_metadata if isinstance(post.raw_metadata, dict) else {}
|
||||
channel = raw.get("channel")
|
||||
if not isinstance(channel, str):
|
||||
return None
|
||||
return channel.strip() or None
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Date the posts whose record reached the disk but never the database (#4436).
|
||||
|
||||
A native walk writes each post's record (`_post.json`, or Discord's
|
||||
`<day>_<message id>_post.json`) as it goes, and phase 3 upserts those records
|
||||
after the walk — which is when the post, and through it its images, get their
|
||||
date. Until #4436 the walk marked a record's post key seen at write time, so a
|
||||
walk killed before phase 3 (a restart, a stall) left the post undated with the
|
||||
ledger saying it was done, and no later tick walked back that far to fix it.
|
||||
|
||||
The record files are still on disk. This finds each undated native post's
|
||||
record under its artist's folder and upserts it with the post's OWN source —
|
||||
never the (artist, platform) lookup, which picks the artist's first source and
|
||||
would misfile a Discord channel's post (#4435). Idempotent: a post that is
|
||||
already dated is never looked at, and upserting a record twice changes nothing.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import Artist, Post, Source
|
||||
from ..utils.sidecar import parse_sidecar
|
||||
from .download_backends import NATIVE_INGESTER_PLATFORMS
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _is_record(path: Path) -> bool:
|
||||
return path.name == "_post.json" or path.name.endswith("_post.json")
|
||||
|
||||
|
||||
def _record_id(path: Path) -> str | None:
|
||||
try:
|
||||
data = json.loads(path.read_text("utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
return parse_sidecar(data).external_post_id
|
||||
|
||||
|
||||
def date_posts_from_records(session: Session, importer, images_root: Path) -> dict:
|
||||
"""Upsert the on-disk record of every undated native post. Returns counts."""
|
||||
rows = session.execute(
|
||||
select(Post.external_post_id, Post.source_id, Post.artist_id, Source.platform)
|
||||
.join(Source, Source.id == Post.source_id)
|
||||
.where(
|
||||
Post.post_date.is_(None),
|
||||
Post.synthesized_by.is_(None),
|
||||
Source.platform.in_(NATIVE_INGESTER_PLATFORMS),
|
||||
)
|
||||
).all()
|
||||
wanted: dict[tuple[int, str], dict[str, int]] = defaultdict(dict)
|
||||
for epid, source_id, artist_id, platform in rows:
|
||||
wanted[(artist_id, platform)][epid] = source_id
|
||||
|
||||
summary = {"undated": len(rows), "dated": 0, "no_record": 0}
|
||||
for (artist_id, platform), by_epid in wanted.items():
|
||||
artist = session.get(Artist, artist_id)
|
||||
root = Path(images_root) / artist.slug / platform if artist else None
|
||||
if root is None or not root.is_dir():
|
||||
summary["no_record"] += len(by_epid)
|
||||
continue
|
||||
found = 0
|
||||
for record in root.rglob("*post.json"):
|
||||
if not _is_record(record):
|
||||
continue
|
||||
epid = _record_id(record)
|
||||
source_id = by_epid.pop(epid, None) if epid else None
|
||||
if source_id is None:
|
||||
continue
|
||||
source = session.get(Source, source_id)
|
||||
if importer.upsert_post_record(record, artist=artist, source=source):
|
||||
found += 1
|
||||
if not by_epid:
|
||||
break
|
||||
summary["dated"] += found
|
||||
summary["no_record"] += len(by_epid)
|
||||
if summary["undated"]:
|
||||
log.info("date_posts_from_records: %s", summary)
|
||||
return summary
|
||||
@@ -19,7 +19,7 @@ from ..models import (
|
||||
)
|
||||
from .db_helpers import failing_sources_clause
|
||||
from .download_backends import uses_native_ingester
|
||||
from .gallery_dl import ErrorType
|
||||
from .download_result import ErrorType
|
||||
from .membership_reconcile import KEPT_KEY, STOPPED_KEY
|
||||
from .membership_roster import gated_reasons_for_sources
|
||||
from .platforms import known_platform_keys
|
||||
@@ -68,6 +68,9 @@ class SourceRecord:
|
||||
artist_slug: str
|
||||
platform: str
|
||||
url: str
|
||||
# alembic 0115: the name the platform gives it, where the URL is opaque
|
||||
# (a Discord link is two ids). None until a walk has read it.
|
||||
display_name: str | None
|
||||
enabled: bool
|
||||
config_overrides: dict | None
|
||||
last_checked_at: str | None
|
||||
@@ -110,6 +113,7 @@ class SourceRecord:
|
||||
"artist_slug": self.artist_slug,
|
||||
"platform": self.platform,
|
||||
"url": self.url,
|
||||
"display_name": self.display_name,
|
||||
"enabled": self.enabled,
|
||||
"config_overrides": self.config_overrides,
|
||||
"last_checked_at": self.last_checked_at,
|
||||
@@ -301,6 +305,7 @@ class SourceService:
|
||||
artist_slug=artist.slug,
|
||||
platform=source.platform,
|
||||
url=source.url,
|
||||
display_name=source.display_name,
|
||||
enabled=source.enabled,
|
||||
config_overrides=source.config_overrides,
|
||||
last_checked_at=source.last_checked_at.isoformat() if source.last_checked_at else None,
|
||||
|
||||
@@ -32,7 +32,6 @@ import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from html import unescape
|
||||
from pathlib import Path
|
||||
@@ -43,6 +42,7 @@ import requests
|
||||
from ..utils.paths import filehash_from_url
|
||||
from .native_ingest_common import (
|
||||
_MAX_429_RETRIES,
|
||||
MediaItem,
|
||||
Membership,
|
||||
NativeAuthError,
|
||||
NativeDriftError,
|
||||
@@ -163,24 +163,6 @@ class SubscribeStarDriftError(SubscribeStarAPIError, NativeDriftError):
|
||||
silently importing nothing."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class MediaItem:
|
||||
"""One resolved downloadable item belonging to a SubscribeStar post.
|
||||
|
||||
Fields mirror patreon_client.MediaItem (so the downloader is structurally the
|
||||
same) plus `media_id` — the stable per-upload gallery id. SubscribeStar's
|
||||
full-res URL is an opaque `/post_uploads?payload=...` (not content-addressed),
|
||||
so `filehash` is usually None and the ledger keys on `<post_id>:<media_id>`.
|
||||
"""
|
||||
|
||||
url: str
|
||||
filename: str
|
||||
kind: str
|
||||
filehash: str | None
|
||||
post_id: str
|
||||
media_id: str
|
||||
|
||||
|
||||
def _extr(text: str, start: str, end: str) -> str:
|
||||
"""Substring between the first `start` and the next `end` after it (gallery-
|
||||
dl's `text.extr`); '' when either marker is absent."""
|
||||
|
||||
@@ -23,30 +23,23 @@ FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
from .native_ingest_common import (
|
||||
BaseNativeDownloader,
|
||||
MediaOutcome,
|
||||
PostRecordOutcome,
|
||||
post_dir_name,
|
||||
sanitize_segment,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SubscribeStarDownloader(BaseNativeDownloader):
|
||||
"""Download resolved SubscribeStar media to gallery-dl's on-disk layout.
|
||||
Subclasses BaseNativeDownloader for the shared streaming GET (transient-retry +
|
||||
Range-resume) and validation/quarantine. No video branch (SubscribeStar serves
|
||||
files directly via /post_uploads) and no detail-fetch (the body is already in
|
||||
the feed HTML). PURE: no DB."""
|
||||
The download loop, per-file layout and sidecars are BaseNativeDownloader's
|
||||
defaults, unchanged: there is no video branch (SubscribeStar serves files
|
||||
directly via /post_uploads) and no detail-fetch (the body is already in the
|
||||
feed HTML). This class adds only the post record. PURE: no DB."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -62,110 +55,8 @@ class SubscribeStarDownloader(BaseNativeDownloader):
|
||||
validate=validate, rate_limit=rate_limit, session=session,
|
||||
)
|
||||
|
||||
# -- public ------------------------------------------------------------
|
||||
|
||||
def download_post(
|
||||
self,
|
||||
post: dict,
|
||||
media_items: list,
|
||||
artist_slug: str,
|
||||
*,
|
||||
is_seen: Callable[[object], bool] = lambda m: False,
|
||||
should_stop: Callable[[], bool] = lambda: False,
|
||||
recapture: bool = False,
|
||||
) -> list[MediaOutcome]:
|
||||
"""Download every media item of one post; return per-item outcomes.
|
||||
Mirrors PatreonDownloader.download_post (two-tier skip, mid-post time-box,
|
||||
recapture surfacing) minus the video branch."""
|
||||
post_dir = self.images_root / artist_slug / "subscribestar" / post_dir_name(post)
|
||||
outcomes: list[MediaOutcome] = []
|
||||
for i, media in enumerate(media_items, start=1):
|
||||
if should_stop():
|
||||
break
|
||||
try:
|
||||
outcomes.append(
|
||||
self._download_one(
|
||||
post, media, post_dir, artist_slug, i, is_seen,
|
||||
recapture=recapture,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # resilient: isolate one item's failure
|
||||
log.warning(
|
||||
"SubscribeStar media failed (post %s, item %d): %s",
|
||||
post.get("id"), i, exc,
|
||||
)
|
||||
outcomes.append(
|
||||
MediaOutcome(media=media, status="error", path=None, error=str(exc))
|
||||
)
|
||||
return outcomes
|
||||
|
||||
# -- per-item ----------------------------------------------------------
|
||||
|
||||
def _download_one(
|
||||
self,
|
||||
post: dict,
|
||||
media,
|
||||
post_dir: Path,
|
||||
artist_slug: str,
|
||||
index: int,
|
||||
is_seen: Callable[[object], bool],
|
||||
*,
|
||||
recapture: bool = False,
|
||||
) -> MediaOutcome:
|
||||
seen = is_seen(media)
|
||||
if seen and not recapture:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
nn = f"{index:02d}"
|
||||
media_path = post_dir / sanitize_segment(f"{nn}_{media.filename}")
|
||||
|
||||
if media_path.exists(): # tier-2: already on disk
|
||||
return MediaOutcome(
|
||||
media=media, status="skipped_disk", path=media_path, error=None
|
||||
)
|
||||
# recapture: a seen item not on disk is NOT re-downloaded (recovery's job).
|
||||
if seen:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
post_dir.mkdir(parents=True, exist_ok=True)
|
||||
if self._rate_limit > 0:
|
||||
time.sleep(self._rate_limit)
|
||||
|
||||
out_path = self._fetch_get(media.url, media_path)
|
||||
reason, quarantine_dest = self._validate_path(out_path, artist_slug, media.url)
|
||||
if reason is not None:
|
||||
return MediaOutcome(
|
||||
media=media, status="quarantined", path=quarantine_dest, error=reason,
|
||||
)
|
||||
self._write_sidecar(post, out_path, source_url=media.url)
|
||||
return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
|
||||
|
||||
# The plain-GET streaming path (_fetch_get / _fetch_to_file) and
|
||||
# _validate_path are inherited from BaseNativeDownloader.
|
||||
|
||||
# -- sidecar -----------------------------------------------------------
|
||||
|
||||
def _write_sidecar(
|
||||
self, post: dict, media_path: Path, *, source_url: str | None = None
|
||||
) -> Path:
|
||||
"""Per-media sidecar — post-first (#856): image identity ONLY
|
||||
(category/id/source_url). The post body/links live solely in _post.json."""
|
||||
return self._write_sidecar_data(
|
||||
post, media_path.with_suffix(".json"), source_url=source_url, minimal=True,
|
||||
)
|
||||
|
||||
def _write_sidecar_data(
|
||||
self, post: dict, sidecar_path: Path, *, source_url: str | None = None,
|
||||
minimal: bool = False,
|
||||
) -> Path:
|
||||
"""Serialize the post's metadata. minimal=True → per-media sidecar (image
|
||||
identity only); else the full post record (body/title/date/url)."""
|
||||
if minimal:
|
||||
data = {"category": "subscribestar", "id": str(post.get("id") or "")}
|
||||
if source_url:
|
||||
data["source_url"] = source_url
|
||||
sidecar_path.write_text(json.dumps(data, indent=2))
|
||||
return sidecar_path
|
||||
def _write_post_record_file(self, post: dict, sidecar_path: Path) -> Path:
|
||||
"""Serialize the full post record: body, title and date."""
|
||||
attrs = post.get("attributes") or {}
|
||||
content = attrs.get("content")
|
||||
# SubscribeStar synthesizes the post permalink from the id (matches the
|
||||
@@ -179,8 +70,6 @@ class SubscribeStarDownloader(BaseNativeDownloader):
|
||||
"content": content if isinstance(content, str) else "",
|
||||
"published_at": attrs.get("published_at"),
|
||||
}
|
||||
if source_url:
|
||||
data["source_url"] = source_url
|
||||
sidecar_path.write_text(json.dumps(data, indent=2))
|
||||
return sidecar_path
|
||||
|
||||
@@ -214,7 +103,7 @@ class SubscribeStarDownloader(BaseNativeDownloader):
|
||||
)
|
||||
post_dir = self.images_root / artist_slug / "subscribestar" / post_dir_name(post)
|
||||
post_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = self._write_sidecar_data(post, post_dir / "_post.json")
|
||||
path = self._write_post_record_file(post, post_dir / "_post.json")
|
||||
body = attrs.get("content")
|
||||
body_chars = len(body) if isinstance(body, str) else 0
|
||||
return PostRecordOutcome(
|
||||
|
||||
@@ -22,30 +22,18 @@ from pathlib import Path
|
||||
|
||||
from ..models import SubscribeStarFailedMedia, SubscribeStarSeenMedia
|
||||
from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester
|
||||
from .subscribestar_client import MediaItem, SubscribeStarAPIError, SubscribeStarClient
|
||||
from .native_ingest_common import ledger_key
|
||||
from .subscribestar_client import SubscribeStarAPIError, SubscribeStarClient
|
||||
from .subscribestar_downloader import SubscribeStarDownloader
|
||||
|
||||
__all__ = [
|
||||
"DEAD_LETTER_THRESHOLD",
|
||||
"SubscribeStarIngester",
|
||||
"_ledger_key",
|
||||
"verify_subscribestar_credential",
|
||||
]
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_LEDGER_KEY_MAX = 128
|
||||
|
||||
|
||||
def _ledger_key(media: MediaItem) -> str:
|
||||
"""Stable per-media identity for the cross-run seen-ledger. SubscribeStar's
|
||||
full-res URL is an opaque `/post_uploads?payload=...` (no content hash), so
|
||||
`media.filehash` is normally None and the stable proxy is the gallery item id
|
||||
scoped to its post: `<post_id>:<media_id>`. Bounded to the column width."""
|
||||
if media.filehash:
|
||||
return media.filehash
|
||||
return f"{media.post_id}:{media.media_id}"[:_LEDGER_KEY_MAX]
|
||||
|
||||
|
||||
class SubscribeStarIngester(Ingester):
|
||||
"""Walk a SubscribeStar creator's posts, download unseen media, return a
|
||||
@@ -91,7 +79,7 @@ class SubscribeStarIngester(Ingester):
|
||||
failed_model=SubscribeStarFailedMedia,
|
||||
seen_constraint="uq_subscribestar_seen_media_source_id",
|
||||
failed_constraint="uq_subscribestar_failed_media_source_id",
|
||||
ledger_key=_ledger_key,
|
||||
ledger_key=ledger_key,
|
||||
platform="subscribestar",
|
||||
error_base=SubscribeStarAPIError,
|
||||
# API_DRIFT message phrasing; the base Ingester._failure_result owns
|
||||
|
||||
@@ -27,13 +27,10 @@ from .image_tag_apply import insert_image_tags
|
||||
|
||||
# image_tag.source stamped on title-heuristic WIP tags — distinct from the other
|
||||
# apply sources so provenance stays legible and a future undo can target only these.
|
||||
# HARD tier ("WIP"/"work in progress") is high-precision → trains the wip head.
|
||||
# Only the artist's own "WIP"/"work in progress" label counts — high-precision, so it
|
||||
# trains the wip head. A sketch/doodle/scribble tier (#1474) was retired in milestone
|
||||
# 430: a "sketch" is usually finished art, and its 6k tags flooded the review strip.
|
||||
WIP_TITLE_SOURCE = "wip_title"
|
||||
# SOFT tier (sketch/doodle/scribble, #1474) is LOWER-precision — a finished "sketch"
|
||||
# is often not WIP. This source is PROVISIONAL (in training_data._AUTO_SOURCES) so it
|
||||
# NEVER trains the wip head; a soft-tagged image that also looks like real content is
|
||||
# surfaced by the ring-loud audit for review.
|
||||
WIP_TITLE_SOFT_SOURCE = "wip_title_soft"
|
||||
|
||||
# A standalone "WIP" / "W.I.P" token, or the phrase "work in progress"
|
||||
# (space/underscore/hyphen separated). The letter-boundary lookarounds are what
|
||||
@@ -45,20 +42,10 @@ _WIP_RE = re.compile(
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Soft tier: sketch / doodle / scribble (+ plurals), letter-boundary anchored so
|
||||
# "sketchbook" / "kadoodle" don't trip it. Deliberately conservative — recall is
|
||||
# secondary because the soft source doesn't train the head and the ring-loud audit
|
||||
# catches false positives.
|
||||
_SOFT_WIP_RE = re.compile(
|
||||
r"(?<![A-Za-z])(?:sketch|sketches|doodle|doodles|scribble|scribbles)(?![A-Za-z])",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Coarse SQL prefilters for the backfill sweep — narrow the post scan to rows that
|
||||
# Coarse SQL prefilter for the backfill sweep — narrows the post scan to rows that
|
||||
# COULD match before the precise regex confirms. Case-insensitive ILIKE patterns.
|
||||
# Each MUST stay a SUPERSET of its regex or the sweep would silently miss posts.
|
||||
# It MUST stay a SUPERSET of the regex or the sweep would silently miss posts.
|
||||
WIP_TITLE_SQL_PREFILTER = ("%wip%", "%work%progress%")
|
||||
SOFT_WIP_TITLE_SQL_PREFILTER = ("%sketch%", "%doodle%", "%scribble%")
|
||||
|
||||
# Chunk bulk inserts so a large sweep can't blow past psycopg's 65535-parameter
|
||||
# ceiling (3 params/row → ~21k rows max; 5k stays comfortably under).
|
||||
@@ -66,19 +53,12 @@ _INSERT_CHUNK = 5000
|
||||
|
||||
|
||||
def matches_wip_title(title: str | None) -> bool:
|
||||
"""True when a post title explicitly marks it work-in-progress (HARD tier)."""
|
||||
"""True when a post title explicitly marks it work-in-progress."""
|
||||
if not title:
|
||||
return False
|
||||
return _WIP_RE.search(title) is not None
|
||||
|
||||
|
||||
def matches_soft_wip_title(title: str | None) -> bool:
|
||||
"""True when a title carries a SOFT WIP cue (sketch/doodle/scribble, #1474)."""
|
||||
if not title:
|
||||
return False
|
||||
return _SOFT_WIP_RE.search(title) is not None
|
||||
|
||||
|
||||
def resolve_wip_tag_id(session: Session) -> int | None:
|
||||
"""The seeded ``wip`` system tag's id (migration 0075), or None if absent."""
|
||||
return session.execute(
|
||||
|
||||
@@ -74,6 +74,12 @@ def prune_missing_file_records_task(self) -> dict:
|
||||
thumbs: list[str] = []
|
||||
last_id = 0
|
||||
with SessionLocal() as session:
|
||||
# While a restore is fetching the library back, files are missing
|
||||
# because they haven't arrived yet, not because they are gone.
|
||||
from .backup import restore_going
|
||||
if restore_going(session):
|
||||
return {"checked": 0, "missing": 0, "deleted": 0,
|
||||
"skipped": "a restore is fetching the library's files"}
|
||||
while True:
|
||||
rows = session.execute(
|
||||
select(ImageRecord.id, ImageRecord.path, ImageRecord.thumbnail_path)
|
||||
@@ -459,3 +465,29 @@ def reclaim_orphaned_attachments_task(self, dry_run: bool = True) -> dict:
|
||||
return cleanup_service.reclaim_orphaned_attachments(
|
||||
session, images_root=IMAGES_ROOT, dry_run=dry_run,
|
||||
)
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.admin.import_unimported_files_task",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError),
|
||||
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||
# One walk of the artists' folders: a stat per file, so minutes on a large
|
||||
# library on network storage. The limits are the backstop for a wedged
|
||||
# filesystem, not the expected exit.
|
||||
soft_time_limit=1800, time_limit=2400, # 30 min / 40 min
|
||||
)
|
||||
def import_unimported_files_task(self, dry_run: bool = True) -> dict:
|
||||
"""Find files in the artists' folders that the library has no record of.
|
||||
dry_run (the default) returns what was found, per artist, without changing
|
||||
anything; apply queues them as a `library` import batch, which attaches each
|
||||
file where it lies and removes the ones the library already holds or the
|
||||
import filters reject."""
|
||||
from ..services import library_scan
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
if dry_run:
|
||||
_found, summary = library_scan.find_unimported(session, IMAGES_ROOT)
|
||||
return {**summary, "batch_id": None}
|
||||
return library_scan.queue_unimported(session, IMAGES_ROOT)
|
||||
|
||||
+788
-117
@@ -8,17 +8,27 @@ BackupRun.
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
import time
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import select, text, update
|
||||
from sqlalchemy.exc import DBAPIError, OperationalError
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..config import get_config
|
||||
from ..models import BackupRun, ImportSettings
|
||||
from ..services import backup_service
|
||||
from ..services import (
|
||||
backup_destination,
|
||||
backup_media,
|
||||
backup_restore,
|
||||
backup_s3,
|
||||
backup_service,
|
||||
backup_verify,
|
||||
)
|
||||
from ._sync_engine import get_sync_engine
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -49,11 +59,19 @@ def _mark_failed(session, row: BackupRun, exc: BaseException) -> None:
|
||||
)
|
||||
def backup_db_task(self, *, tag: str | None = None,
|
||||
triggered_by: str = "manual") -> dict:
|
||||
"""Create one DB backup. Returns {'backup_run_id': N}."""
|
||||
"""Create one DB backup. Returns {'backup_run_id': N}.
|
||||
|
||||
With an off-site destination set up, the dump streams to that bucket and
|
||||
nothing is written here; the row's manifest says where it went
|
||||
(`offsite`: endpoint, bucket, key). Without one, the dump is written to
|
||||
/images/_backups."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
cfg = get_config()
|
||||
now = datetime.now(UTC)
|
||||
with SessionLocal() as session:
|
||||
# A restore replaces the database this would dump.
|
||||
if restore_going(session):
|
||||
return {"skipped": "a restore is going"}
|
||||
row = BackupRun(
|
||||
kind="db", status="running", tag=tag,
|
||||
triggered_by=triggered_by, started_at=now, manifest={},
|
||||
@@ -64,10 +82,17 @@ def backup_db_task(self, *, tag: str | None = None,
|
||||
run_id = row.id
|
||||
|
||||
try:
|
||||
result = backup_service.backup_db(
|
||||
db_url=cfg.database_url_sync, images_root=IMAGES_ROOT,
|
||||
tag=tag, triggered_by=triggered_by,
|
||||
)
|
||||
with SessionLocal() as session:
|
||||
settings = ImportSettings.load_sync(session)
|
||||
target = backup_destination.saved_target(settings)
|
||||
limit_mbps = settings.backup_upload_limit_mbps
|
||||
if target is not None:
|
||||
result = _dump_offsite(cfg, target, limit_mbps)
|
||||
else:
|
||||
result = backup_service.backup_db(
|
||||
db_url=cfg.database_url_sync, images_root=IMAGES_ROOT,
|
||||
tag=tag, triggered_by=triggered_by,
|
||||
)
|
||||
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, run_id)
|
||||
@@ -79,61 +104,23 @@ def backup_db_task(self, *, tag: str | None = None,
|
||||
row = session.get(BackupRun, run_id)
|
||||
row.status = "ok"
|
||||
row.finished_at = datetime.now(UTC)
|
||||
row.sql_path = result["sql_path"]
|
||||
row.size_bytes = result["size_bytes"]
|
||||
row.manifest = {
|
||||
"manifest_path": result["manifest_path"],
|
||||
"ts": result["ts"],
|
||||
}
|
||||
session.commit()
|
||||
return {"backup_run_id": run_id}
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.backup.backup_images_task",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError),
|
||||
retry_backoff=30, retry_backoff_max=300, max_retries=1,
|
||||
soft_time_limit=21600, time_limit=23400,
|
||||
)
|
||||
def backup_images_task(self, *, tag: str | None = None,
|
||||
triggered_by: str = "manual") -> dict:
|
||||
"""Create one images backup. Same shape as backup_db_task; uses
|
||||
tar_path instead of sql_path."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
now = datetime.now(UTC)
|
||||
with SessionLocal() as session:
|
||||
row = BackupRun(
|
||||
kind="images", status="running", tag=tag,
|
||||
triggered_by=triggered_by, started_at=now, manifest={},
|
||||
)
|
||||
session.add(row)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
run_id = row.id
|
||||
|
||||
try:
|
||||
result = backup_service.backup_images(
|
||||
images_root=IMAGES_ROOT,
|
||||
tag=tag, triggered_by=triggered_by,
|
||||
)
|
||||
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, run_id)
|
||||
if row is not None:
|
||||
_mark_failed(session, row, exc)
|
||||
raise
|
||||
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, run_id)
|
||||
row.status = "ok"
|
||||
row.finished_at = datetime.now(UTC)
|
||||
row.tar_path = result["tar_path"]
|
||||
row.size_bytes = result["size_bytes"]
|
||||
row.manifest = {
|
||||
"manifest_path": result["manifest_path"],
|
||||
"ts": result["ts"],
|
||||
}
|
||||
if target is not None:
|
||||
row.manifest = {
|
||||
"ts": result["ts"],
|
||||
"offsite": {
|
||||
"endpoint": result["endpoint"],
|
||||
"bucket": result["bucket"],
|
||||
"key": result["key"],
|
||||
"parts": result["parts"],
|
||||
},
|
||||
}
|
||||
else:
|
||||
row.sql_path = result["sql_path"]
|
||||
row.manifest = {
|
||||
"manifest_path": result["manifest_path"],
|
||||
"ts": result["ts"],
|
||||
}
|
||||
session.commit()
|
||||
return {"backup_run_id": run_id}
|
||||
|
||||
@@ -189,52 +176,6 @@ def restore_db_task(self, *, source_backup_run_id: int) -> dict:
|
||||
return {"backup_run_id": marker_id}
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.backup.restore_images_task",
|
||||
bind=True, max_retries=0,
|
||||
soft_time_limit=21600, time_limit=23400,
|
||||
)
|
||||
def restore_images_task(self, *, source_backup_run_id: int) -> dict:
|
||||
"""Mirrors restore_db_task; uses backup_service.restore_images."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
now = datetime.now(UTC)
|
||||
with SessionLocal() as session:
|
||||
src = session.get(BackupRun, source_backup_run_id)
|
||||
if src is None or src.kind != "images" or not src.tar_path:
|
||||
raise ValueError(
|
||||
f"BackupRun id={source_backup_run_id} is not a valid images backup"
|
||||
)
|
||||
marker = BackupRun(
|
||||
kind="images", status="restoring",
|
||||
triggered_by="restore", started_at=now,
|
||||
restored_from_id=src.id,
|
||||
manifest={"source_tar_path": src.tar_path},
|
||||
)
|
||||
session.add(marker)
|
||||
session.commit()
|
||||
session.refresh(marker)
|
||||
marker_id = marker.id
|
||||
tar_path = src.tar_path
|
||||
|
||||
try:
|
||||
backup_service.restore_images(
|
||||
images_root=IMAGES_ROOT, tar_path=Path(tar_path),
|
||||
)
|
||||
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, marker_id)
|
||||
if row is not None:
|
||||
_mark_failed(session, row, exc)
|
||||
raise
|
||||
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, marker_id)
|
||||
row.status = "restored"
|
||||
row.finished_at = datetime.now(UTC)
|
||||
session.commit()
|
||||
return {"backup_run_id": marker_id}
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.backup.prune_backups",
|
||||
soft_time_limit=300, time_limit=600,
|
||||
@@ -242,34 +183,39 @@ def restore_images_task(self, *, source_backup_run_id: int) -> dict:
|
||||
def prune_backups() -> dict:
|
||||
"""Daily Beat. Per-kind retention from ImportSettings.
|
||||
|
||||
Returns {"db_deleted": N, "images_deleted": M, "files_unlinked": K}.
|
||||
Returns {"db_deleted": N, "media_deleted": L, "restore_deleted": R, "reconcile_deleted": C, "test_deleted": T,
|
||||
"files_unlinked": K}. Media, restore, reconcile and test runs leave no
|
||||
files here; only their history is trimmed, to the latest MEDIA_RUNS_KEPT,
|
||||
RESTORE_RUNS_KEPT, RECONCILE_RUNS_KEPT and TEST_RUNS_KEPT.
|
||||
Tagged rows (tag IS NOT NULL) are never pruned.
|
||||
Status='running' / 'restoring' rows are never pruned — the
|
||||
recover_stalled_backup_runs sweep flips truly-stuck ones to
|
||||
'error' first. (Earlier docstring claimed the FC-3i TaskRun sweep
|
||||
handled those, but TaskRun cleanup never touched BackupRun rows.
|
||||
Audit 2026-06-02 added the dedicated sweep.)
|
||||
'error' first.
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
counts = {"db_deleted": 0, "images_deleted": 0, "files_unlinked": 0}
|
||||
counts = {"db_deleted": 0, "media_deleted": 0,
|
||||
"restore_deleted": 0, "reconcile_deleted": 0, "test_deleted": 0,
|
||||
"files_unlinked": 0}
|
||||
with SessionLocal() as session:
|
||||
s = ImportSettings.load_sync(session)
|
||||
for kind, keep in (
|
||||
("db", s.backup_db_keep_last_n),
|
||||
("images", s.backup_images_keep_last_n),
|
||||
("media", MEDIA_RUNS_KEPT),
|
||||
("restore", RESTORE_RUNS_KEPT),
|
||||
("reconcile", RECONCILE_RUNS_KEPT),
|
||||
("test", TEST_RUNS_KEPT),
|
||||
):
|
||||
candidates = session.execute(
|
||||
select(BackupRun)
|
||||
.where(BackupRun.kind == kind)
|
||||
.where(BackupRun.tag.is_(None))
|
||||
.where(BackupRun.status.in_(["ok", "error"]))
|
||||
.where(BackupRun.status.in_(["ok", "error", "restored"]))
|
||||
.order_by(BackupRun.started_at.desc())
|
||||
.offset(keep)
|
||||
).scalars().all()
|
||||
for row in candidates:
|
||||
result = backup_service.unlink_artifact_files(
|
||||
sql_path=row.sql_path,
|
||||
tar_path=row.tar_path,
|
||||
manifest_path=(row.manifest or {}).get("manifest_path"),
|
||||
)
|
||||
counts["files_unlinked"] += sum(
|
||||
@@ -300,4 +246,729 @@ def backup_db_nightly() -> dict:
|
||||
if now_hour != configured_hour:
|
||||
return {"skipped": f"hour={now_hour} != configured={configured_hour}"}
|
||||
res = backup_db_task.delay(triggered_by="nightly")
|
||||
return {"dispatched": res.id}
|
||||
out = {"dispatched": res.id}
|
||||
# With an off-site destination, the night's backup is the whole library:
|
||||
# the database dump, then any images and attachments not yet in the bucket.
|
||||
# Both run on the long-maintenance lane, one after the other.
|
||||
with SessionLocal() as session:
|
||||
try:
|
||||
offsite = backup_destination.saved_target(ImportSettings.load_sync(session))
|
||||
except Exception: # noqa: BLE001 — the media run reports it itself
|
||||
offsite = True
|
||||
if offsite is not None:
|
||||
out["media_dispatched"] = backup_media_task.delay(triggered_by="nightly").id
|
||||
return out
|
||||
|
||||
|
||||
# ── Off-site copy of the library's files ─────────────────────────────────────
|
||||
|
||||
# Each task invocation sends files for this long, then queues the next chunk
|
||||
# of the same run, so other long-maintenance jobs get their turn in between.
|
||||
MEDIA_CHUNK_SECONDS = 45 * 60
|
||||
MEDIA_RUNS_KEPT = 60
|
||||
# A failed file is listed on the run, up to this many; the count covers all.
|
||||
MEDIA_FAILURES_LISTED = 20
|
||||
# The weekly bucket check keeps about a quarter's history.
|
||||
RECONCILE_RUNS_KEPT = 12
|
||||
|
||||
|
||||
def _dump_offsite(cfg, target, limit_mbps) -> dict:
|
||||
"""Dump the database to the bucket, with its row counts beside it for a
|
||||
test restore to check against. The dump stands without its counts, so a
|
||||
failure to store them is logged, not raised."""
|
||||
result = backup_service.backup_db_offsite(
|
||||
db_url=cfg.database_url_sync, target=target, limit_mbps=limit_mbps)
|
||||
if result.get("counts"):
|
||||
try:
|
||||
backup_verify.store_counts(target, result["key"], result["counts"])
|
||||
except (backup_s3.S3Error, OSError):
|
||||
log.warning("backup: the row counts for %s weren't stored", result["key"],
|
||||
exc_info=True)
|
||||
return result
|
||||
|
||||
|
||||
def _offsite_manifest(target) -> dict:
|
||||
return {"endpoint": target.endpoint, "bucket": target.bucket,
|
||||
"key": backup_media.PREFIX}
|
||||
|
||||
|
||||
# ── One chunk at a time ──────────────────────────────────────────────────────
|
||||
#
|
||||
# A media backup and a restore's file phase run as a chain of chunks, each
|
||||
# queuing the next. The chunk carrying a run writes its token to the run's
|
||||
# manifest as "chunk", and hands that token to the chunk it queues as `after`.
|
||||
# A chunk queued to resume a run after a restart carries `resume` instead and
|
||||
# takes the run over. Any other message is stale (a killed chunk's own message
|
||||
# delivered again, say) and is dropped, so a run never goes on twice and the
|
||||
# chain never doubles.
|
||||
|
||||
|
||||
def _chunk_token(task) -> str:
|
||||
return task.request.id or uuid.uuid4().hex
|
||||
|
||||
|
||||
def _claim_run(session, run_id: int, status: str, token: str, *, after: str | None,
|
||||
resume: bool) -> str | None:
|
||||
"""Take the run for this chunk. Returns None when taken, or why not."""
|
||||
row = session.get(BackupRun, run_id, with_for_update=True)
|
||||
if row is None or row.status != status:
|
||||
session.rollback()
|
||||
return "the run is no longer going"
|
||||
held = (row.manifest or {}).get("chunk")
|
||||
if not resume and held is not None and held != after:
|
||||
session.rollback()
|
||||
return "another chunk carries this run on"
|
||||
row.manifest = {**(row.manifest or {}), "chunk": token}
|
||||
session.commit()
|
||||
return None
|
||||
|
||||
|
||||
def _still_ours(session, run_id: int, token: str) -> bool:
|
||||
"""Lock the run's row and say whether this chunk still carries it. A chunk
|
||||
that was taken over stops at its next file rather than write over the
|
||||
newer chunk's place; on False the transaction is already rolled back."""
|
||||
held = session.execute(
|
||||
select(BackupRun.manifest).where(BackupRun.id == run_id).with_for_update()
|
||||
).scalar_one_or_none()
|
||||
if (held or {}).get("chunk") == token:
|
||||
return True
|
||||
session.rollback()
|
||||
return False
|
||||
|
||||
|
||||
def resume_offsite_runs() -> dict:
|
||||
"""Queue the next chunk of every off-site run a restart cut short: a media
|
||||
backup still running, and a restore fetching its files. Called when the
|
||||
long-maintenance lane starts, where those chunks run, so none of theirs is
|
||||
alive then; the queued chunk carries on from the run's saved place. A
|
||||
restore stopped while loading its database isn't resumed: the stall sweep
|
||||
ends it, and the screen offers the restore again."""
|
||||
with _sync_session_factory()() as session:
|
||||
media = list(session.execute(
|
||||
select(BackupRun.id).where(BackupRun.kind == "media")
|
||||
.where(BackupRun.status == "running")
|
||||
).scalars())
|
||||
restores = [
|
||||
r.id for r in session.execute(
|
||||
select(BackupRun).where(BackupRun.kind == "restore")
|
||||
.where(BackupRun.status == "restoring")
|
||||
).scalars()
|
||||
if (r.manifest or {}).get("phase") == "files"
|
||||
]
|
||||
try:
|
||||
tests = end_cut_off_tests()
|
||||
except Exception: # noqa: BLE001 — resuming the runs below matters more
|
||||
log.exception("couldn't end a test restore cut off by the restart")
|
||||
tests = 0
|
||||
for run_id in media:
|
||||
backup_media_task.delay(run_id=run_id, resume=True)
|
||||
for run_id in restores:
|
||||
restore_offsite_task.delay(run_id=run_id, resume=True)
|
||||
if media or restores:
|
||||
log.info("resumed off-site runs after a restart: media %s, restore %s",
|
||||
media, restores)
|
||||
return {"media": media, "restore": restores, "tests_ended": tests}
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.backup.backup_media_task",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError),
|
||||
retry_backoff=30, retry_backoff_max=300, max_retries=2,
|
||||
# A chunk stops starting files after MEDIA_CHUNK_SECONDS; the limits leave
|
||||
# room for the file in flight, a multi-gigabyte archive at a low upload
|
||||
# limit included.
|
||||
soft_time_limit=21600, time_limit=23400,
|
||||
)
|
||||
def backup_media_task(self, *, run_id: int | None = None, triggered_by: str = "manual",
|
||||
after: str | None = None, resume: bool = False) -> dict:
|
||||
"""Send the library's images and attachments that aren't in the off-site
|
||||
bucket yet, one chunk of a run. Called without `run_id` it starts a run
|
||||
(unless one is already going); each chunk queues the next until nothing is
|
||||
left, and a restart resumes the run where it stopped. A file that can't be
|
||||
sent is listed on the run and tried again by the next run. Returns
|
||||
{'backup_run_id': N, ...}."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
now = datetime.now(UTC)
|
||||
token = _chunk_token(self)
|
||||
with SessionLocal() as session:
|
||||
if run_id is None:
|
||||
going = session.execute(
|
||||
select(BackupRun.id).where(BackupRun.kind == "media")
|
||||
.where(BackupRun.status == "running").limit(1)
|
||||
).scalar_one_or_none()
|
||||
if going is not None:
|
||||
return {"backup_run_id": going, "skipped": "a media backup is already running"}
|
||||
# During a restore the library's files are still arriving.
|
||||
if restore_going(session):
|
||||
return {"skipped": "a restore is going"}
|
||||
row = BackupRun(kind="media", status="running", triggered_by=triggered_by,
|
||||
started_at=now, last_progress_at=now, size_bytes=0,
|
||||
manifest={"sent": 0, "failed": 0, "failures": [], "cursor": "",
|
||||
"chunk": token})
|
||||
session.add(row)
|
||||
session.commit()
|
||||
run_id = row.id
|
||||
else:
|
||||
why = _claim_run(session, run_id, "running", token, after=after, resume=resume)
|
||||
if why:
|
||||
return {"backup_run_id": run_id, "skipped": why}
|
||||
|
||||
try:
|
||||
with SessionLocal() as session:
|
||||
settings = ImportSettings.load_sync(session)
|
||||
target = backup_destination.saved_target(settings)
|
||||
limit_mbps = settings.backup_upload_limit_mbps
|
||||
if target is None:
|
||||
raise backup_s3.S3Error("No off-site destination is set up.")
|
||||
more = _send_media_chunk(SessionLocal, run_id, target, backup_s3.Pacer(limit_mbps),
|
||||
token)
|
||||
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, run_id)
|
||||
if row is not None:
|
||||
_mark_failed(session, row, exc)
|
||||
raise
|
||||
|
||||
if more is None:
|
||||
return {"backup_run_id": run_id, "skipped": "another chunk took the run over"}
|
||||
if more:
|
||||
backup_media_task.delay(run_id=run_id, after=token)
|
||||
return {"backup_run_id": run_id, "continues": True}
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, run_id)
|
||||
m = dict(row.manifest or {})
|
||||
row.status = "ok"
|
||||
row.finished_at = datetime.now(UTC)
|
||||
if m.get("failed"):
|
||||
first = (m.get("failures") or [{}])[0]
|
||||
row.error = (f"{m['failed']} file(s) weren't sent; the next run tries them again. "
|
||||
f"First: {first.get('path')}: {first.get('error')}")[:2000]
|
||||
session.commit()
|
||||
return {"backup_run_id": run_id, "continues": False}
|
||||
|
||||
|
||||
def _send_media_chunk(SessionLocal, run_id: int, target, pacer, token: str) -> bool | None:
|
||||
"""Send files until the chunk's time is up or none are left. Progress is
|
||||
committed after every file, so a chunk that dies loses at most the file
|
||||
in flight. Returns True when files remain for another chunk, and None when
|
||||
another chunk has taken the run over."""
|
||||
tid = backup_media.target_id(target)
|
||||
deadline = time.monotonic() + MEDIA_CHUNK_SECONDS
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, run_id)
|
||||
m = dict(row.manifest or {})
|
||||
m["offsite"] = _offsite_manifest(target)
|
||||
cursor = m.get("cursor") or ""
|
||||
while True:
|
||||
batch = backup_media.pending(session, tid, after=cursor)
|
||||
# End the read before any upload starts: a transaction left open
|
||||
# across a long transfer holds back vacuum for its whole length.
|
||||
session.commit()
|
||||
if not batch:
|
||||
if not _still_ours(session, run_id, token):
|
||||
return None
|
||||
m["cursor"] = cursor
|
||||
row.manifest = dict(m)
|
||||
row.last_progress_at = datetime.now(UTC)
|
||||
session.commit()
|
||||
return False
|
||||
for blob in batch:
|
||||
if time.monotonic() > deadline:
|
||||
if not _still_ours(session, run_id, token):
|
||||
return None
|
||||
m["cursor"] = cursor
|
||||
row.manifest = dict(m)
|
||||
session.commit()
|
||||
return True
|
||||
key = None
|
||||
try:
|
||||
key = backup_media.send(target, blob, IMAGES_ROOT, pacer)
|
||||
except (backup_s3.S3Error, OSError) as exc:
|
||||
m["failed"] = m.get("failed", 0) + 1
|
||||
if len(m.get("failures") or []) < MEDIA_FAILURES_LISTED:
|
||||
m["failures"] = [*(m.get("failures") or []),
|
||||
{"path": blob.path, "error": str(exc)[:300]}]
|
||||
if not _still_ours(session, run_id, token):
|
||||
return None
|
||||
if key is not None:
|
||||
backup_media.record(session, tid, [(blob, key)])
|
||||
m["sent"] = m.get("sent", 0) + 1
|
||||
row.size_bytes = (row.size_bytes or 0) + blob.size_bytes
|
||||
cursor = blob.sha256
|
||||
m["cursor"] = cursor
|
||||
row.manifest = dict(m)
|
||||
row.last_progress_at = datetime.now(UTC)
|
||||
session.commit()
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.backup.reconcile_media_backup",
|
||||
soft_time_limit=1800, time_limit=2100,
|
||||
)
|
||||
def reconcile_media_backup() -> dict:
|
||||
"""Weekly Beat. Compare what the off-site bucket holds under `media/` with
|
||||
what FabledCurator believes it sent: a missing object is sent again by
|
||||
the next run, and an object already there is not sent twice. Skipped
|
||||
while a media run is going, and when no destination is set up.
|
||||
|
||||
Each check is kept as a run of kind "reconcile" with its counts
|
||||
({listed, dropped, adopted}), so the Backups card lists it and the System
|
||||
view can warn when files were missing from the bucket."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
target = backup_destination.saved_target(ImportSettings.load_sync(session))
|
||||
if target is None:
|
||||
return {"skipped": "no destination"}
|
||||
going = session.execute(
|
||||
select(BackupRun.id).where(BackupRun.kind == "media")
|
||||
.where(BackupRun.status == "running").limit(1)
|
||||
).scalar_one_or_none()
|
||||
if going is not None:
|
||||
return {"skipped": "a media backup is running"}
|
||||
now = datetime.now(UTC)
|
||||
row = BackupRun(kind="reconcile", status="running", triggered_by="weekly",
|
||||
started_at=now, last_progress_at=now,
|
||||
manifest={"offsite": _offsite_manifest(target)})
|
||||
session.add(row)
|
||||
session.commit()
|
||||
run_id = row.id
|
||||
try:
|
||||
counts = backup_media.reconcile(session, target)
|
||||
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||
session.rollback()
|
||||
row = session.get(BackupRun, run_id)
|
||||
if row is not None:
|
||||
_mark_failed(session, row, exc)
|
||||
raise
|
||||
row = session.get(BackupRun, run_id)
|
||||
row.status = "ok"
|
||||
row.finished_at = datetime.now(UTC)
|
||||
row.manifest = {**(row.manifest or {}), **counts}
|
||||
session.commit()
|
||||
return {"backup_run_id": run_id, **counts}
|
||||
|
||||
|
||||
# ── Restoring from the off-site bucket ───────────────────────────────────────
|
||||
|
||||
# The file phase fetches for this long per task, then queues its next chunk,
|
||||
# like a media run.
|
||||
RESTORE_CHUNK_SECONDS = 45 * 60
|
||||
RESTORE_RUNS_KEPT = 20
|
||||
RESTORE_FAILURES_LISTED = 20
|
||||
# A run over files already on disk notes its place this often, not per file.
|
||||
_RESTORE_NOTE_SECONDS = 5
|
||||
_RESTORE_COUNTS = ("restored", "copied", "present", "missing", "failed")
|
||||
|
||||
|
||||
def backup_busy(session) -> str | None:
|
||||
"""Why a backup or a restore can't start right now, or None."""
|
||||
row = session.execute(
|
||||
select(BackupRun.kind).where(BackupRun.status.in_(["running", "restoring"])).limit(1)
|
||||
).first()
|
||||
if row is None:
|
||||
return None
|
||||
if row.kind == "restore":
|
||||
return "a restore is going"
|
||||
if row.kind == "test":
|
||||
return "a test restore is running"
|
||||
return f"a {row.kind} backup is running"
|
||||
|
||||
|
||||
def restore_going(session) -> bool:
|
||||
return session.execute(
|
||||
select(BackupRun.id).where(BackupRun.kind == "restore")
|
||||
.where(BackupRun.status == "restoring").limit(1)
|
||||
).first() is not None
|
||||
|
||||
|
||||
# ── Test restore ─────────────────────────────────────────────────────────────
|
||||
|
||||
TEST_RUNS_KEPT = 20
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.backup.restore_test_task",
|
||||
bind=True,
|
||||
max_retries=0,
|
||||
# Downloading a dump, loading it and rebuilding its indexes; a restore of
|
||||
# the database alone, with room to spare.
|
||||
soft_time_limit=4 * 3600, time_limit=4 * 3600 + 900,
|
||||
)
|
||||
def restore_test_task(self, *, dump_key: str | None = None,
|
||||
source_backup_run_id: int | None = None) -> dict:
|
||||
"""Test a database dump without touching this install: load it into a
|
||||
temporary database, check every table against the counts recorded with
|
||||
the dump, bring it up to this version, and drop it again
|
||||
(services/backup_verify.py). Tests the bucket's dump at `dump_key` (the
|
||||
newest when None), or, given `source_backup_run_id`, a dump on this
|
||||
server. The run ends "ok" when every table matched, "error" with what
|
||||
didn't otherwise. Returns {'backup_run_id': N, ...}."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
cfg = get_config()
|
||||
now = datetime.now(UTC)
|
||||
with SessionLocal() as session:
|
||||
if why := restore_test_blocked(session):
|
||||
return {"skipped": why}
|
||||
row = BackupRun(kind="test", status="running", triggered_by="manual",
|
||||
started_at=now, last_progress_at=now, manifest={"phase": "fetching"})
|
||||
session.add(row)
|
||||
session.commit()
|
||||
run_id = row.id
|
||||
|
||||
def note(phase: str, **more) -> None:
|
||||
with SessionLocal() as session:
|
||||
r = session.get(BackupRun, run_id)
|
||||
_note(session, r, {**r.manifest, "phase": phase, **more})
|
||||
|
||||
staged = offsite = None
|
||||
try:
|
||||
if source_backup_run_id is not None:
|
||||
dump_path, recorded, source = _local_dump(SessionLocal, source_backup_run_id)
|
||||
else:
|
||||
with SessionLocal() as session:
|
||||
settings = ImportSettings.load_sync(session)
|
||||
target = backup_destination.saved_target(settings)
|
||||
limit_mbps = settings.backup_upload_limit_mbps
|
||||
if target is None:
|
||||
raise backup_s3.S3Error("No off-site destination is set up.")
|
||||
dump = backup_restore.find_dump(target, dump_key)
|
||||
source = {"key": dump.key, "ts": dump.ts, "size_bytes": dump.size_bytes}
|
||||
offsite = {**_offsite_manifest(target), "key": dump.key}
|
||||
note("fetching", dump=source, offsite=offsite)
|
||||
recorded = backup_verify.fetch_counts(target, dump.key)
|
||||
staged = backup_restore.fetch_dump(
|
||||
target, dump, IMAGES_ROOT / "_backups", backup_s3.Pacer(limit_mbps))
|
||||
dump_path = staged
|
||||
note("loading", dump=source)
|
||||
result = backup_verify.trial_restore(
|
||||
db_url=cfg.database_url_sync, dump_path=dump_path, recorded=recorded,
|
||||
note=note)
|
||||
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||
with SessionLocal() as session:
|
||||
r = session.get(BackupRun, run_id)
|
||||
if r is not None:
|
||||
_mark_failed(session, r, exc)
|
||||
raise
|
||||
finally:
|
||||
if staged is not None:
|
||||
staged.unlink(missing_ok=True)
|
||||
|
||||
with SessionLocal() as session:
|
||||
r = session.get(BackupRun, run_id)
|
||||
r.status = "error" if result["mismatched"] else "ok"
|
||||
r.error = backup_verify.summary(result) if result["mismatched"] else None
|
||||
r.finished_at = datetime.now(UTC)
|
||||
r.size_bytes = source.get("size_bytes")
|
||||
r.manifest = {"phase": "done", "dump": source, "offsite": offsite,
|
||||
"summary": backup_verify.summary(result), **result}
|
||||
session.commit()
|
||||
return {"backup_run_id": run_id, "mismatched": result["mismatched"]}
|
||||
|
||||
|
||||
def _local_dump(SessionLocal, run_id: int) -> tuple[Path, dict | None, dict]:
|
||||
"""A dump on this server: its path, its recorded counts, and what the
|
||||
test run says it tested."""
|
||||
with SessionLocal() as session:
|
||||
src = session.get(BackupRun, run_id)
|
||||
if src is None or src.kind != "db" or src.status != "ok" or not src.sql_path:
|
||||
raise backup_s3.S3Error(f"Backup {run_id} isn't a database dump on this server.")
|
||||
path = Path(src.sql_path)
|
||||
if not path.is_file():
|
||||
raise backup_s3.S3Error(f"{path} is no longer on this server.")
|
||||
recorded = backup_verify.local_counts((src.manifest or {}).get("manifest_path"))
|
||||
ts = (src.manifest or {}).get("ts")
|
||||
return path, recorded, {"run_id": run_id, "ts": ts, "size_bytes": src.size_bytes}
|
||||
|
||||
|
||||
def restore_test_blocked(session) -> str | None:
|
||||
"""Why a test restore can't start right now, or None. A backup going
|
||||
doesn't stop one: the test only reads this install's database."""
|
||||
if restore_going(session):
|
||||
return "a restore is going"
|
||||
going = session.execute(
|
||||
select(BackupRun.id).where(BackupRun.kind == "test")
|
||||
.where(BackupRun.status == "running").limit(1)
|
||||
).first()
|
||||
return "a test restore is already running" if going else None
|
||||
|
||||
|
||||
def end_cut_off_tests() -> int:
|
||||
"""End every test restore a restart cut short, and drop the temporary
|
||||
database it left. Called when the long-maintenance lane starts, where the
|
||||
tests run, so none is alive then."""
|
||||
with _sync_session_factory()() as session:
|
||||
ended = session.execute(
|
||||
update(BackupRun).where(BackupRun.kind == "test")
|
||||
.where(BackupRun.status == "running")
|
||||
.values(status="error", finished_at=datetime.now(UTC),
|
||||
error="Interrupted: the server restarted during the test. Test again.")
|
||||
).rowcount or 0
|
||||
session.commit()
|
||||
if ended:
|
||||
backup_verify.drop_scratch(get_config().database_url_sync)
|
||||
return ended
|
||||
|
||||
|
||||
def _library_is_empty(session) -> bool:
|
||||
return session.execute(text(
|
||||
"SELECT NOT EXISTS (SELECT 1 FROM image_record) "
|
||||
"AND NOT EXISTS (SELECT 1 FROM post_attachment)"
|
||||
)).scalar()
|
||||
|
||||
|
||||
def _note(session, row: BackupRun, m: dict, token: str | None = None) -> bool:
|
||||
"""Write the run's progress. Given a chunk's token, writes only while that
|
||||
chunk still carries the run, and returns False when it doesn't."""
|
||||
if token is not None and not _still_ours(session, row.id, token):
|
||||
return False
|
||||
row.manifest = dict(m)
|
||||
row.last_progress_at = datetime.now(UTC)
|
||||
session.commit()
|
||||
return True
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.backup.restore_offsite_task",
|
||||
bind=True,
|
||||
max_retries=0, # a half-applied restore is never retried on its own
|
||||
soft_time_limit=21600, time_limit=23400,
|
||||
)
|
||||
def restore_offsite_task(self, *, dump_key: str | None = None, files_only: bool = False,
|
||||
run_id: int | None = None, after: str | None = None,
|
||||
resume: bool = False) -> dict:
|
||||
"""Restore this install from the off-site bucket, one chunk at a time.
|
||||
|
||||
Started without `run_id`, it replaces the database with the dump at
|
||||
`dump_key` (the newest when None), then fetches every library file the
|
||||
restored rows name. With `files_only`, the database stays and only the
|
||||
missing files are fetched, which also repairs a library that lost files.
|
||||
Each chunk queues the next, and a restart resumes the file phase where it
|
||||
stopped; the run's row counts files restored from the bucket, copied from
|
||||
a duplicate already on disk, already present, missing from the bucket,
|
||||
and failed. Returns {'backup_run_id': N, ...}."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
token = _chunk_token(self)
|
||||
if run_id is not None:
|
||||
return _restore_files(SessionLocal, run_id, token, after=after, resume=resume)
|
||||
|
||||
now = datetime.now(UTC)
|
||||
with SessionLocal() as session:
|
||||
busy = backup_busy(session)
|
||||
if busy:
|
||||
return {"skipped": busy}
|
||||
settings = ImportSettings.load_sync(session)
|
||||
limit_mbps = settings.backup_upload_limit_mbps
|
||||
row = BackupRun(kind="restore", status="restoring", triggered_by="restore",
|
||||
started_at=now, last_progress_at=now, size_bytes=0,
|
||||
manifest={"phase": "files" if files_only else "database",
|
||||
"cursor": "", "failures": [],
|
||||
**dict.fromkeys(_RESTORE_COUNTS, 0)})
|
||||
session.add(row)
|
||||
session.commit()
|
||||
run_id = row.id
|
||||
try:
|
||||
target = backup_destination.saved_target(settings)
|
||||
if target is None:
|
||||
raise backup_s3.S3Error("No off-site destination is set up.")
|
||||
except backup_s3.S3Error as exc:
|
||||
_mark_failed(session, row, exc)
|
||||
raise
|
||||
row.manifest = {**row.manifest, "offsite": _offsite_manifest(target)}
|
||||
session.commit()
|
||||
|
||||
if files_only:
|
||||
return _restore_files(SessionLocal, run_id, token, after=None, resume=False)
|
||||
run_id = _restore_database(SessionLocal, run_id, target, limit_mbps, dump_key)
|
||||
# The files follow as chunks of their own, on the worker.
|
||||
restore_offsite_task.delay(run_id=run_id)
|
||||
return {"backup_run_id": run_id, "continues": True}
|
||||
|
||||
|
||||
def _restore_database(SessionLocal, run_id: int, target, limit_mbps, dump_key) -> int:
|
||||
"""Replace the database with a dump from the bucket. Returns the run's id
|
||||
in the restored database, where its row is written again."""
|
||||
cfg = get_config()
|
||||
pacer = backup_s3.Pacer(limit_mbps)
|
||||
staged = None
|
||||
try:
|
||||
dump = backup_restore.find_dump(target, dump_key)
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, run_id)
|
||||
m = dict(row.manifest)
|
||||
m["dump"] = {"key": dump.key, "ts": dump.ts, "size_bytes": dump.size_bytes}
|
||||
# The database about to be replaced goes to the bucket first, so
|
||||
# the restore can itself be undone. An empty one isn't worth a
|
||||
# dump, and would sit in the list as the newest.
|
||||
if not _library_is_empty(session):
|
||||
session.commit()
|
||||
saved = _dump_offsite(cfg, target, limit_mbps)
|
||||
m["saved_before"] = saved["key"]
|
||||
_note(session, row, m)
|
||||
started_at, manifest = row.started_at, dict(m)
|
||||
staged = backup_restore.fetch_dump(target, dump, IMAGES_ROOT / "_backups", pacer)
|
||||
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||
if staged is not None:
|
||||
staged.unlink(missing_ok=True)
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, run_id)
|
||||
if row is not None:
|
||||
_mark_failed(session, row, exc)
|
||||
raise
|
||||
|
||||
# From here on the database is being replaced; every pooled connection
|
||||
# still points at the tables being dropped.
|
||||
get_sync_engine().dispose()
|
||||
try:
|
||||
backup_service.restore_db(db_url=cfg.database_url_sync, sql_path=staged)
|
||||
backup_restore.migrate()
|
||||
backup_service.end_other_connections(db_url=cfg.database_url_sync)
|
||||
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||
_leave_empty_install(SessionLocal, cfg, target, limit_mbps, started_at, manifest, exc)
|
||||
raise
|
||||
finally:
|
||||
staged.unlink(missing_ok=True)
|
||||
get_sync_engine().dispose()
|
||||
|
||||
with SessionLocal() as session:
|
||||
backup_restore.keep_destination(session, target, limit_mbps)
|
||||
# The dump holds its own run as still going: it is the dump just
|
||||
# restored, so it finished. Any other run going then is over.
|
||||
dump_at = datetime.strptime(manifest["dump"]["ts"], "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC)
|
||||
own = session.execute(
|
||||
select(BackupRun).where(BackupRun.kind == "db", BackupRun.status == "running")
|
||||
# The name's time is truncated to the second, after the row began.
|
||||
.where(BackupRun.started_at.between(dump_at - timedelta(minutes=2),
|
||||
dump_at + timedelta(seconds=1)))
|
||||
.order_by(BackupRun.started_at.desc()).limit(1)
|
||||
).scalar_one_or_none()
|
||||
if own is not None:
|
||||
own.status = "ok"
|
||||
own.manifest = {"ts": manifest["dump"]["ts"], "offsite": {
|
||||
**manifest["offsite"], "key": manifest["dump"]["key"]}}
|
||||
own.size_bytes = manifest["dump"]["size_bytes"]
|
||||
session.execute(
|
||||
update(BackupRun).where(BackupRun.status.in_(["running", "restoring"]))
|
||||
.values(status="error", finished_at=datetime.now(UTC),
|
||||
error="Interrupted: this run was going when the restored dump was taken.")
|
||||
)
|
||||
row = BackupRun(kind="restore", status="restoring", triggered_by="restore",
|
||||
started_at=started_at, last_progress_at=datetime.now(UTC),
|
||||
size_bytes=0, manifest={**manifest, "phase": "files"})
|
||||
session.add(row)
|
||||
session.commit()
|
||||
return row.id
|
||||
|
||||
|
||||
def _leave_empty_install(SessionLocal, cfg, target, limit_mbps, started_at, manifest,
|
||||
exc: BaseException) -> None:
|
||||
"""After a failed load, put back an empty database at this version, so the
|
||||
install starts and can try again, and record why it stopped."""
|
||||
try:
|
||||
backup_service.wipe_db(db_url=cfg.database_url_sync)
|
||||
backup_restore.migrate()
|
||||
backup_service.end_other_connections(db_url=cfg.database_url_sync)
|
||||
get_sync_engine().dispose()
|
||||
with SessionLocal() as session:
|
||||
backup_restore.keep_destination(session, target, limit_mbps)
|
||||
session.add(BackupRun(
|
||||
kind="restore", status="error", triggered_by="restore",
|
||||
started_at=started_at, finished_at=datetime.now(UTC), manifest=manifest,
|
||||
error=(f"The restore stopped while loading the database: {exc}. "
|
||||
"The database is empty again; choose a dump and restore again.")[:2000],
|
||||
))
|
||||
session.commit()
|
||||
except Exception: # noqa: BLE001 — the original failure is the one raised
|
||||
log.exception("restore: the database couldn't be reset after a failed load")
|
||||
|
||||
|
||||
def _restore_files(SessionLocal, run_id: int, token: str, *, after: str | None,
|
||||
resume: bool) -> dict:
|
||||
"""One chunk of a restore's file phase; queues the next while files remain."""
|
||||
with SessionLocal() as session:
|
||||
why = _claim_run(session, run_id, "restoring", token, after=after, resume=resume)
|
||||
if why:
|
||||
return {"backup_run_id": run_id, "skipped": why}
|
||||
row = session.get(BackupRun, run_id)
|
||||
settings = ImportSettings.load_sync(session)
|
||||
limit_mbps = settings.backup_upload_limit_mbps
|
||||
try:
|
||||
target = backup_destination.saved_target(settings)
|
||||
if target is None:
|
||||
raise backup_s3.S3Error("No off-site destination is set up.")
|
||||
except backup_s3.S3Error as exc:
|
||||
_mark_failed(session, row, exc)
|
||||
raise
|
||||
try:
|
||||
more = _fetch_files_chunk(SessionLocal, run_id, target, backup_s3.Pacer(limit_mbps),
|
||||
token)
|
||||
except (SoftTimeLimitExceeded, Exception) as exc:
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, run_id)
|
||||
if row is not None:
|
||||
_mark_failed(session, row, exc)
|
||||
raise
|
||||
|
||||
if more is None:
|
||||
return {"backup_run_id": run_id, "skipped": "another chunk took the run over"}
|
||||
if more:
|
||||
restore_offsite_task.delay(run_id=run_id, after=token)
|
||||
return {"backup_run_id": run_id, "continues": True}
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, run_id)
|
||||
m = dict(row.manifest or {})
|
||||
row.status = "restored"
|
||||
row.finished_at = datetime.now(UTC)
|
||||
missed = m.get("missing", 0) + m.get("failed", 0)
|
||||
if missed:
|
||||
first = (m.get("failures") or [{}])[0]
|
||||
row.error = (f"{m.get('missing', 0)} file(s) aren't in the bucket and "
|
||||
f"{m.get('failed', 0)} couldn't be restored. "
|
||||
f"First: {first.get('path')}: {first.get('error')}")[:2000]
|
||||
session.commit()
|
||||
# Thumbnails aren't backed up: they are made again from the restored files.
|
||||
from .thumbnail import backfill_thumbnails
|
||||
backfill_thumbnails.delay()
|
||||
return {"backup_run_id": run_id, "continues": False}
|
||||
|
||||
|
||||
def _fetch_files_chunk(SessionLocal, run_id: int, target, pacer, token: str) -> bool | None:
|
||||
"""Put library files back until the chunk's time is up or none are left.
|
||||
Returns True when files remain for another chunk, and None when another
|
||||
chunk has taken the run over."""
|
||||
tid = backup_media.target_id(target)
|
||||
deadline = time.monotonic() + RESTORE_CHUNK_SECONDS
|
||||
with SessionLocal() as session:
|
||||
row = session.get(BackupRun, run_id)
|
||||
m = dict(row.manifest or {})
|
||||
cursor = m.get("cursor") or ""
|
||||
noted = time.monotonic()
|
||||
while True:
|
||||
batch = backup_restore.library_files(session, after=cursor)
|
||||
keys = backup_restore.sent_keys(session, tid, [f.sha256 for f in batch])
|
||||
# End the read before any download starts.
|
||||
session.commit()
|
||||
if not batch:
|
||||
m["cursor"] = cursor
|
||||
return False if _note(session, row, m, token) else None
|
||||
for f in batch:
|
||||
if time.monotonic() > deadline:
|
||||
m["cursor"] = cursor
|
||||
return True if _note(session, row, m, token) else None
|
||||
try:
|
||||
outcome = backup_restore.restore_file(
|
||||
target, f, keys.get(f.sha256), IMAGES_ROOT, pacer)
|
||||
except (backup_s3.S3Error, OSError) as exc:
|
||||
outcome = "missing" if isinstance(exc, backup_s3.ObjectMissing) else "failed"
|
||||
if len(m.get("failures") or []) < RESTORE_FAILURES_LISTED:
|
||||
m["failures"] = [*(m.get("failures") or []),
|
||||
{"path": f.paths[0], "error": str(exc)[:300]}]
|
||||
m[outcome] = m.get(outcome, 0) + 1
|
||||
if outcome in ("restored", "copied"):
|
||||
row.size_bytes = (row.size_bytes or 0) + f.size_bytes
|
||||
cursor = f.sha256
|
||||
m["cursor"] = cursor
|
||||
if outcome != "present" or time.monotonic() - noted > _RESTORE_NOTE_SECONDS:
|
||||
if not _note(session, row, m, token):
|
||||
return None
|
||||
noted = time.monotonic()
|
||||
|
||||
+128
-41
@@ -24,7 +24,7 @@ from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import redis
|
||||
from sqlalchemy import delete, select, update
|
||||
from sqlalchemy import delete, or_, select, update
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..config import get_config
|
||||
@@ -32,6 +32,7 @@ from ..models import Artist, ExternalLink, ImageRecord, ImportSettings, Post, So
|
||||
from ..services.external_fetch import fetch_external
|
||||
from ..services.importer import Importer
|
||||
from ..services.thumbnailer import Thumbnailer
|
||||
from ..utils.staging import is_temp_staging_file
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -39,14 +40,20 @@ log = logging.getLogger(__name__)
|
||||
IMAGES_ROOT = Path("/images")
|
||||
|
||||
# After this many failed attempts a link is dead-lettered (skipped by routine
|
||||
# sweeps; an operator recovery still re-attempts). Mirrors the ingester ledger.
|
||||
# sweeps; the daily recovery retries it, sparingly). Mirrors the ingester ledger.
|
||||
DEAD_LETTER_THRESHOLD = 3
|
||||
# Per-fetch read + total budgets now live in external_fetch (a short read
|
||||
# timeout fails a stalled host fast; a generous total caps a big-but-flowing
|
||||
# download). The celery soft/hard time_limit is the outer backstop above those.
|
||||
# Links enqueued per sweep — bounds the burst when a big backfill records many.
|
||||
_SWEEP_BATCH = 50
|
||||
# Dead rows older than this are pruned (retention).
|
||||
# Recovery gives a dead link one more attempt once it has been dead for
|
||||
# 2**recoveries days (1, 2, 4, 8, 16), and stops after _MAX_RECOVERIES. A host
|
||||
# outage gets several later chances; a link that can never succeed (too big to
|
||||
# import in time, a file made private) costs a handful of downloads, not three
|
||||
# a day.
|
||||
_MAX_RECOVERIES = 5
|
||||
# A link recovery has given up on is pruned this long after its last attempt.
|
||||
_RETENTION_DAYS = 30
|
||||
# Per-host serialize lock. TTL is a safety net for a worker that dies holding it
|
||||
# (normal completion/error releases it in `finally`); sized just past the fetch
|
||||
@@ -54,6 +61,9 @@ _RETENTION_DAYS = 30
|
||||
# than one fetch would have taken.
|
||||
_LOCK_PREFIX = "fc:extdl_lock:"
|
||||
_LOCK_TTL = 2400
|
||||
# A link 'downloading' this long has outlived fetch_external_link's hard limit
|
||||
# (time_limit=3600), so its worker is gone and the sweep recovers it.
|
||||
_STALL_MINUTES = 65
|
||||
_SERIALIZE_COUNTDOWN = 120
|
||||
_MAX_SERIALIZE_WAITS = 30
|
||||
|
||||
@@ -106,7 +116,7 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
|
||||
ExternalLink.id == link_id,
|
||||
ExternalLink.status.in_(("pending", "failed")),
|
||||
)
|
||||
.values(status="downloading")
|
||||
.values(status="downloading", started_at=datetime.now(UTC))
|
||||
.returning(ExternalLink.id)
|
||||
).first()
|
||||
session.commit()
|
||||
@@ -180,6 +190,9 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
|
||||
/ str(post.external_post_id) / str(link_id)
|
||||
)
|
||||
try:
|
||||
# Clear the staging files an interrupted attempt left behind. Complete
|
||||
# files it left stay, and are imported once this fetch succeeds.
|
||||
_discard_partials(post_dir)
|
||||
result = fetch_external(host, url, post_dir)
|
||||
with SessionLocal() as session:
|
||||
link = session.get(ExternalLink, link_id)
|
||||
@@ -190,6 +203,7 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
|
||||
)
|
||||
_record_failure(link, attempts, result.error or "fetch failed")
|
||||
session.commit()
|
||||
_discard_partials(post_dir)
|
||||
return {"link_id": link_id, "error": result.error}
|
||||
|
||||
log.info(
|
||||
@@ -201,7 +215,8 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
|
||||
post = session.get(Post, link.post_id)
|
||||
artist = session.get(Artist, post.artist_id)
|
||||
source = session.get(Source, post.source_id) if post.source_id else None
|
||||
image_ids = _route_files(session, result.files, post, platform, artist, source)
|
||||
files = _unimported_files(session, post_dir, first=result.files)
|
||||
image_ids = _route_files(session, files, post, platform, artist, source)
|
||||
link.status = "downloaded"
|
||||
link.last_error = None
|
||||
link.completed_at = datetime.now(UTC)
|
||||
@@ -231,6 +246,7 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
|
||||
if link is not None and link.status == "downloading":
|
||||
_record_failure(link, attempts, f"{type(exc).__name__}: {exc}")
|
||||
session.commit()
|
||||
_discard_partials(post_dir)
|
||||
raise
|
||||
finally:
|
||||
if lock is not None:
|
||||
@@ -287,10 +303,9 @@ def _route_files(session, files, post, platform, artist, source) -> list[int]:
|
||||
post.id, f.name, result.status, len(ids),
|
||||
)
|
||||
elif result.status == "attached":
|
||||
# Non-art / archive captured as a PostAttachment (copied into the
|
||||
# store) — drop the on-disk original. An archive captured WITHOUT
|
||||
# extracting any image carries the reason (the recurring "zip but no
|
||||
# images" symptom, #718) — surface it loudly.
|
||||
# Non-art, or an archive with no new images, captured as a
|
||||
# PostAttachment. An archive captured WITHOUT extracting any image
|
||||
# carries the reason, so the "zip but no images" case is visible.
|
||||
if result.error:
|
||||
log.warning(
|
||||
"extdl archive captured UNEXTRACTED: post=%s file=%s reason=%s",
|
||||
@@ -298,51 +313,104 @@ def _route_files(session, files, post, platform, artist, source) -> list[int]:
|
||||
)
|
||||
else:
|
||||
log.info("extdl attachment captured: post=%s file=%s", post.id, f.name)
|
||||
f.unlink(missing_ok=True)
|
||||
elif result.status == "skipped":
|
||||
reason = result.skip_reason.value if result.skip_reason else None
|
||||
log.info(
|
||||
"extdl skipped: post=%s file=%s reason=%s", post.id, f.name, reason,
|
||||
)
|
||||
if reason in ("duplicate_hash", "duplicate_phash"):
|
||||
# Path-safe unlink: external files are imported IN PLACE, so `f`
|
||||
# can BE the canonical record's backing file (same content
|
||||
# re-fetched / a colliding name). Only delete `f` when it is NOT
|
||||
# the existing record's file — otherwise we orphan the record and
|
||||
# playback 404s. (#859)
|
||||
canonical = None
|
||||
if result.image_id is not None:
|
||||
rec = session.get(ImageRecord, result.image_id)
|
||||
if rec is not None:
|
||||
canonical = (IMAGES_ROOT / rec.path).resolve()
|
||||
if canonical != f.resolve():
|
||||
f.unlink(missing_ok=True) # a redundant copy — safe to drop
|
||||
else:
|
||||
log.info(
|
||||
"extdl keep: %s IS record %s's canonical file — not unlinking",
|
||||
f.name, result.image_id,
|
||||
)
|
||||
elif result.status == "failed":
|
||||
log.warning(
|
||||
"extdl import FAILED: post=%s file=%s error=%s",
|
||||
post.id, f.name, result.error,
|
||||
)
|
||||
f.unlink(missing_ok=True)
|
||||
else: # 'refreshed' or any future status — work happened, no new images
|
||||
log.info(
|
||||
"extdl import: post=%s file=%s status=%s", post.id, f.name, result.status,
|
||||
)
|
||||
# Same policy as the subscription download path: drop the file unless
|
||||
# the library needs it at this path. A file imported in place is kept.
|
||||
importer.discard_after_attach(f, result)
|
||||
# The synthesized sidecar has done its job — don't litter the tree.
|
||||
Path(str(f) + ".json").unlink(missing_ok=True)
|
||||
return image_ids
|
||||
|
||||
|
||||
def _unimported_files(session, link_dir: Path, *, first=()) -> list[Path]:
|
||||
"""Every complete file under a link's dir, subfolders included, that no
|
||||
ImageRecord points at: what this fetch wrote plus anything an earlier
|
||||
attempt finished writing but never imported. `first` (the fetch's own
|
||||
files) leads the list. Staging files and sidecars are left out."""
|
||||
if not link_dir.is_dir():
|
||||
return []
|
||||
found = {
|
||||
p for p in link_dir.rglob("*")
|
||||
if p.is_file() and not is_temp_staging_file(p) and p.suffix.lower() != ".json"
|
||||
}
|
||||
ordered = [p for p in first if p in found] + sorted(found.difference(first))
|
||||
recorded = set(session.execute(
|
||||
select(ImageRecord.path).where(
|
||||
ImageRecord.path.in_([str(p) for p in ordered])
|
||||
)
|
||||
).scalars()) if ordered else set()
|
||||
return [p for p in ordered if str(p) not in recorded]
|
||||
|
||||
|
||||
def _discard_partials(link_dir: Path) -> int:
|
||||
"""Delete the staging files under a link's dir (megatools' `.megatmp.*`, an
|
||||
HTTP fetch's `.part`), then the dirs left empty. Every fetcher writes to a
|
||||
staging name and renames on completion, so a staging file is a transfer
|
||||
that never finished. Complete files stay: they may be the only copy of art
|
||||
whose upstream has since gone. Returns the number of files removed."""
|
||||
if not link_dir.is_dir():
|
||||
return 0
|
||||
removed = 0
|
||||
for p in link_dir.rglob("*"):
|
||||
if not (p.is_file() and is_temp_staging_file(p)):
|
||||
continue
|
||||
try:
|
||||
p.unlink(missing_ok=True)
|
||||
removed += 1
|
||||
except OSError as exc:
|
||||
log.warning("extdl could not remove %s: %s", p, exc)
|
||||
for d in sorted((p for p in link_dir.rglob("*") if p.is_dir()), reverse=True):
|
||||
try:
|
||||
d.rmdir()
|
||||
except OSError:
|
||||
pass # not empty: it holds a complete file
|
||||
try:
|
||||
link_dir.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
if removed:
|
||||
log.info("extdl removed %d leftover file(s) from %s", removed, link_dir)
|
||||
return removed
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.external.sweep_external_links")
|
||||
def sweep_external_links() -> dict:
|
||||
"""Enqueue a bounded batch of actionable links (pending, or failed below the
|
||||
dead-letter threshold). Driven by the post-download hook and a beat tick."""
|
||||
dead-letter threshold). Driven by the post-download hook and a beat tick.
|
||||
|
||||
First it recovers links whose worker died mid-fetch: a link 'downloading'
|
||||
for longer than the task's hard limit has no worker left, so it counts as a
|
||||
failed attempt and goes back to the retry path (dead-lettered at the
|
||||
threshold like any other failure). The next attempt clears its staging
|
||||
files and imports the complete ones."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
now = datetime.now(UTC)
|
||||
stale = session.execute(
|
||||
select(ExternalLink).where(
|
||||
ExternalLink.status == "downloading",
|
||||
or_(
|
||||
ExternalLink.started_at.is_(None),
|
||||
ExternalLink.started_at < now - timedelta(minutes=_STALL_MINUTES),
|
||||
),
|
||||
)
|
||||
).scalars().all()
|
||||
for link in stale:
|
||||
_record_failure(link, link.attempts, "worker stopped mid-fetch")
|
||||
session.commit()
|
||||
ids = session.execute(
|
||||
select(ExternalLink.id).where(
|
||||
ExternalLink.status.in_(("pending", "failed")),
|
||||
@@ -351,34 +419,53 @@ def sweep_external_links() -> dict:
|
||||
).scalars().all()
|
||||
for link_id in ids:
|
||||
fetch_external_link.delay(link_id)
|
||||
return {"enqueued": len(ids)}
|
||||
return {"enqueued": len(ids), "recovered": len(stale)}
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.external.recover_external_links")
|
||||
def recover_external_links() -> dict:
|
||||
"""Recovery sweep (rule #89): reset dead links back to retryable so a stuck
|
||||
host outage or a since-fixed bug gets another pass, then enqueue. Bounded."""
|
||||
"""Recovery sweep: give each dead link one more attempt once it
|
||||
has been dead for 2**recoveries days, then enqueue. A recovered link that
|
||||
fails again goes straight back to dead, and after _MAX_RECOVERIES it stays
|
||||
dead until retention prunes it."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
session.execute(
|
||||
update(ExternalLink).where(ExternalLink.status == "dead")
|
||||
.values(status="failed", attempts=0, last_error=None)
|
||||
)
|
||||
now = datetime.now(UTC)
|
||||
candidates = session.execute(
|
||||
select(ExternalLink).where(
|
||||
ExternalLink.status == "dead",
|
||||
ExternalLink.recoveries < _MAX_RECOVERIES,
|
||||
)
|
||||
).scalars().all()
|
||||
recovered = 0
|
||||
for link in candidates:
|
||||
wait = timedelta(days=2 ** link.recoveries)
|
||||
if link.completed_at is not None and link.completed_at > now - wait:
|
||||
continue
|
||||
link.status = "failed"
|
||||
link.attempts = DEAD_LETTER_THRESHOLD - 1
|
||||
link.recoveries += 1
|
||||
recovered += 1
|
||||
session.commit()
|
||||
return sweep_external_links()
|
||||
return {**sweep_external_links(), "revived": recovered}
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.external.prune_external_links")
|
||||
def prune_external_links() -> dict:
|
||||
"""Retention (rule #89): delete long-dead links so the ledger doesn't grow
|
||||
unbounded. Downloaded links are kept (they're the record of what we have)."""
|
||||
"""Retention: delete links recovery has given up on, once their
|
||||
last attempt is _RETENTION_DAYS old, so the ledger doesn't grow unbounded.
|
||||
Downloaded links are kept (they're the record of what we have)."""
|
||||
cutoff = datetime.now(UTC) - timedelta(days=_RETENTION_DAYS)
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
res = session.execute(
|
||||
delete(ExternalLink).where(
|
||||
ExternalLink.status == "dead",
|
||||
ExternalLink.created_at < cutoff,
|
||||
ExternalLink.recoveries >= _MAX_RECOVERIES,
|
||||
or_(
|
||||
ExternalLink.completed_at.is_(None),
|
||||
ExternalLink.completed_at < cutoff,
|
||||
),
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
@@ -19,7 +19,8 @@ from sqlalchemy.exc import DBAPIError, OperationalError
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..models import ImportBatch, ImportSettings, ImportTask
|
||||
from ..services.importer import Importer
|
||||
from ..services.importer import Importer, ImportResult
|
||||
from ..services.library_scan import folder_owner
|
||||
from ..services.thumbnailer import Thumbnailer
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
@@ -45,8 +46,13 @@ def _mark_failed(session, task, error_msg: str) -> None:
|
||||
"""Best-effort flip of a 'processing' row to 'failed' + batch counter
|
||||
increment. Wrapped in its own try because if the DB is what just
|
||||
broke, this commit will also fail — that's why the maintenance sweep
|
||||
exists as a backstop."""
|
||||
exists as a backstop.
|
||||
|
||||
The failure being recorded may have left the session's transaction dead
|
||||
(an error mid-flush), so it is rolled back first; otherwise this commit
|
||||
fails on it too and the row stays 'processing'."""
|
||||
try:
|
||||
session.rollback()
|
||||
task.status = "failed"
|
||||
task.error = error_msg
|
||||
task.finished_at = datetime.now(UTC)
|
||||
@@ -165,6 +171,27 @@ def enqueue_import(task_id: int, task_type: str) -> None:
|
||||
import_media_file.delay(task_id)
|
||||
|
||||
|
||||
def _attach_where_it_lies(session, importer: Importer, path: Path) -> ImportResult:
|
||||
"""Import a file that already sits in its artist's folder (a `library`
|
||||
batch, from services.library_scan). It is attached in place with that
|
||||
artist and source. The post record its folder holds (`_post.json`) is filed
|
||||
first, so the post gets its title and date. Afterwards the file is removed
|
||||
if the library didn't keep it: a copy of something it already holds, or a
|
||||
file the import filters reject. A file whose import crashed stays, with the
|
||||
task marked failed: it may be the only copy of an archive the extractor
|
||||
can't read yet."""
|
||||
if not path.is_file():
|
||||
return ImportResult(status="skipped", error="no longer on disk")
|
||||
artist, source = folder_owner(session, path, IMAGES_ROOT)
|
||||
record = path.parent / "_post.json"
|
||||
if artist is not None and record.is_file():
|
||||
importer.upsert_post_record(record, artist=artist, source=source)
|
||||
result = importer.attach_in_place(path, artist=artist, source=source)
|
||||
if result.status != "failed":
|
||||
importer.discard_after_attach(path, result)
|
||||
return result
|
||||
|
||||
|
||||
def _do_import(session, task, import_task_id: int) -> dict:
|
||||
"""Actual work, called from inside the resilience wrapper."""
|
||||
settings = ImportSettings.load_sync(session)
|
||||
@@ -180,7 +207,10 @@ def _do_import(session, task, import_task_id: int) -> dict:
|
||||
deep=deep,
|
||||
)
|
||||
|
||||
result = importer.import_one(Path(task.source_path))
|
||||
if batch is not None and batch.scan_mode == "library":
|
||||
result = _attach_where_it_lies(session, importer, Path(task.source_path))
|
||||
else:
|
||||
result = importer.import_one(Path(task.source_path))
|
||||
|
||||
if result.status in ("imported", "superseded"):
|
||||
task.status = "complete"
|
||||
|
||||
@@ -20,10 +20,12 @@ from ..models import (
|
||||
ImportSettings,
|
||||
ImportTask,
|
||||
LibraryAuditRun,
|
||||
PostAttachment,
|
||||
Source,
|
||||
TaskRun,
|
||||
)
|
||||
from ..utils.phash import compute_phash
|
||||
from ..utils.staging import is_temp_staging_file
|
||||
from ._sync_engine import get_sync_engine
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
@@ -81,13 +83,10 @@ WIP_BACKFILL_PAGE = 500
|
||||
FFPROBE_TIMEOUT_SECONDS = 10
|
||||
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
||||
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
||||
# Orphaned staging files: downloads/imports stage into <name>.part / <name>.partial
|
||||
# then os.replace() into place (importer / external_fetch / native_ingest_common /
|
||||
# attachment_store), so a kill mid-write leaves a discardable temp, never a corrupt
|
||||
# final. cleanup_orphaned_temp_files sweeps ones left behind; the min-age guard
|
||||
# keeps it from deleting an in-flight download's staging file mid-write.
|
||||
# Orphaned staging files (utils.staging): cleanup_orphaned_temp_files sweeps the
|
||||
# ones a killed write left behind; the min-age guard keeps it from deleting an
|
||||
# in-flight download's staging file mid-write.
|
||||
IMAGES_ROOT = Path("/images")
|
||||
TEMP_STAGING_SUFFIXES = (".part", ".partial")
|
||||
ORPHAN_TEMP_MIN_AGE_HOURS = 6
|
||||
|
||||
# Audit 2026-06-02: per-entity recovery sweep thresholds. Each must be
|
||||
@@ -137,7 +136,13 @@ IMPORT_BATCH_KEEP_DAYS = 30
|
||||
# (the import queue itself stays at the 5-min default for single
|
||||
# files); time_limit=2100.
|
||||
QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||
"ml": 25,
|
||||
# ml: the scheduled auto-apply sweeps and refresh_character_prototypes run
|
||||
# to a 35-min hard limit (2100s); 25 swept them mid-run (#4432). The two
|
||||
# 65-min jobs have their own entries below.
|
||||
"ml": 40,
|
||||
# import: import_media_file's hard limit is 6 min (360s), one past the
|
||||
# 5-min default this queue fell to (#4432).
|
||||
"import": 10,
|
||||
# download_source legitimately walks 5-25 min (Patreon/gallery-dl
|
||||
# deep creators); its hard time_limit is DOWNLOAD_HARD_TIME_LIMIT
|
||||
# (1500s = 25m). The 5-min default flagged healthy in-flight walks as
|
||||
@@ -154,14 +159,23 @@ QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||
# overrides below cover the outliers (backups, library audit).
|
||||
"maintenance": 75,
|
||||
"scan": 75,
|
||||
# The long lane (#4432). Until TaskRun.queue asked the router, nothing was
|
||||
# recorded here: these runs read as `maintenance` (75) or, for
|
||||
# translation, `default` (5 — which failed healthy 35-min runs). The
|
||||
# longest task without its own entry below is the admin family at a
|
||||
# 40-min hard limit; 45 = 40 + 5.
|
||||
"maintenance_long": 45,
|
||||
}
|
||||
TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||
"backend.app.tasks.import_file.import_archive_file": 40,
|
||||
# Backup images runs hours, not minutes (6.5h hard limit). The
|
||||
# task-name override beats the queue's 75-min default so a
|
||||
# legitimately-running backup isn't flagged.
|
||||
"backend.app.tasks.backup.backup_images_task": 420,
|
||||
"backend.app.tasks.backup.restore_images_task": 420,
|
||||
# One media-backup chunk stops starting files after 45 min but may finish
|
||||
# a large file in flight, up to its 6.5 h hard limit.
|
||||
"backend.app.tasks.backup.backup_media_task": 420,
|
||||
# A restore's database load, or one chunk of its file phase, likewise.
|
||||
"backend.app.tasks.backup.restore_offsite_task": 420,
|
||||
# A test restore downloads a dump and loads it whole; 4 h 15 min hard
|
||||
# limit. 260 = 255 + 5.
|
||||
"backend.app.tasks.backup.restore_test_task": 260,
|
||||
# Library audit scans the full library — 2h hard limit.
|
||||
"backend.app.tasks.library_audit.scan_library_for_rule": 130,
|
||||
# External file-host fetches (mega/gdrive/film packs) can run to the task's
|
||||
@@ -179,9 +193,33 @@ TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||
# external-fetch entry above — without an override a healthy in-flight walk
|
||||
# is swept 'RecoverySweep' at the bare 5-min default. 30 = 25 + 5.
|
||||
"backend.app.tasks.admin.reclaim_orphaned_attachments_task": 30,
|
||||
# Importing what's on disk stats every file in the artists' folders, and
|
||||
# its hard limit is 40 min for the same wedged-filesystem case. 45 = 40 + 5.
|
||||
"backend.app.tasks.admin.import_unimported_files_task": 45,
|
||||
# Head training and the manual head apply run to 65 min (3900s) — past the
|
||||
# ml queue's threshold (#4432). 70 = 65 + 5.
|
||||
"backend.app.tasks.ml.train_heads": 70,
|
||||
"backend.app.tasks.ml.apply_head_tags": 70,
|
||||
}
|
||||
|
||||
|
||||
def _import_queue_depth() -> int | None:
|
||||
"""Messages waiting in the broker's import queue, or None when the broker
|
||||
can't be read."""
|
||||
try:
|
||||
import redis
|
||||
|
||||
from ..config import get_config
|
||||
|
||||
client = redis.Redis.from_url(
|
||||
get_config().celery_broker_url, socket_timeout=5, socket_connect_timeout=5,
|
||||
)
|
||||
return int(client.llen("import"))
|
||||
except Exception: # noqa: BLE001 — an unreadable broker means "don't know"
|
||||
log.warning("recover_interrupted_tasks: import queue depth unreadable", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
|
||||
def recover_interrupted_tasks() -> int:
|
||||
"""Recover stuck ImportTask rows. Two distinct stuck states:
|
||||
@@ -197,16 +235,22 @@ def recover_interrupted_tasks() -> int:
|
||||
2026-05-24 after a 2224-row zombie pile; archive split out
|
||||
2026-05-28.)
|
||||
|
||||
2. 'pending' or 'queued' > 30 min — enqueue-phase crash. scan_directory
|
||||
creates rows with status='pending' (commit), then in a second pass
|
||||
transitions to 'queued' and calls .delay() (commit). If the scanner
|
||||
crashes between those two commits, rows are orphaned in 'pending'
|
||||
(never enqueued) with no recovery path — invisible to the
|
||||
'processing' sweep above. Flagged 2026-05-25 by operator hitting a
|
||||
5490-row orphan pile. Flip these to 'failed' (not re-enqueue) so
|
||||
the operator drains them via /api/import/retry-failed at their own
|
||||
pace; bulk-re-enqueueing 5000+ rows would thundering-herd the
|
||||
import worker.
|
||||
2. 'pending' > 30 min — enqueue-phase crash. scan_directory creates rows
|
||||
with status='pending' (commit), then in a second pass transitions to
|
||||
'queued' and calls .delay() (commit). If the scanner crashes between
|
||||
those two commits, rows are orphaned in 'pending' (never enqueued)
|
||||
with no recovery path — invisible to the 'processing' sweep above.
|
||||
Flip these to 'failed' (not re-enqueue) so the operator drains them
|
||||
via /api/import/retry-failed at their own pace; bulk-re-enqueueing
|
||||
5000+ rows would thundering-herd the import worker.
|
||||
|
||||
A 'queued' row has a broker message, and in a batch larger than half
|
||||
an hour of work it is simply waiting its turn, so its age says
|
||||
nothing. It counts as orphaned only when the broker's import queue
|
||||
is empty: then its message is gone (a broker flush or restart) and
|
||||
nothing will ever run it. When the queue can't be read, queued rows
|
||||
are left alone — a queued row costs nothing to keep, and failing a
|
||||
live one loses its import.
|
||||
|
||||
Returns total rows touched (recovered + marked failed).
|
||||
"""
|
||||
@@ -283,9 +327,12 @@ def recover_interrupted_tasks() -> int:
|
||||
)
|
||||
stuck = stuck_result.all()
|
||||
|
||||
orphan_statuses = ["pending"]
|
||||
if _import_queue_depth() == 0:
|
||||
orphan_statuses.append("queued")
|
||||
orphan_result = session.execute(
|
||||
update(ImportTask)
|
||||
.where(ImportTask.status.in_(["pending", "queued"]))
|
||||
.where(ImportTask.status.in_(orphan_statuses))
|
||||
.where(ImportTask.created_at < orphan_cutoff)
|
||||
.values(
|
||||
status="failed",
|
||||
@@ -355,22 +402,58 @@ def cleanup_old_tasks() -> int:
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
def _paths_with_rows(paths: list[str]) -> set[str]:
|
||||
"""The subset of `paths` that an image (its file or thumbnail) or an
|
||||
attachment row points at."""
|
||||
owned: set[str] = set()
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
for i in range(0, len(paths), 500):
|
||||
chunk = paths[i:i + 500]
|
||||
owned.update(session.execute(
|
||||
select(ImageRecord.path).where(ImageRecord.path.in_(chunk))
|
||||
).scalars())
|
||||
owned.update(session.execute(
|
||||
select(ImageRecord.thumbnail_path).where(ImageRecord.thumbnail_path.in_(chunk))
|
||||
).scalars())
|
||||
owned.update(session.execute(
|
||||
select(PostAttachment.path).where(PostAttachment.path.in_(chunk))
|
||||
).scalars())
|
||||
return owned
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.cleanup_orphaned_temp_files")
|
||||
def cleanup_orphaned_temp_files() -> int:
|
||||
"""Delete orphaned .part/.partial staging files under the images root, left by
|
||||
a download/import killed mid-write. Only removes files older than
|
||||
ORPHAN_TEMP_MIN_AGE_HOURS so an in-flight download's staging file is never
|
||||
pulled out from under it. Returns the count removed."""
|
||||
"""Delete orphaned staging files (.part, .partial, megatools' .megatmp.*)
|
||||
under the images root, left by a download/import killed mid-write. Only
|
||||
removes files older than ORPHAN_TEMP_MIN_AGE_HOURS so an in-flight
|
||||
download's staging file is never pulled out from under it, and never a
|
||||
file an image or attachment row points at, whatever its name: that file
|
||||
is library content. Returns the count removed."""
|
||||
if not IMAGES_ROOT.is_dir():
|
||||
return 0
|
||||
cutoff = datetime.now(UTC).timestamp() - ORPHAN_TEMP_MIN_AGE_HOURS * 3600
|
||||
removed = 0
|
||||
stale: list[Path] = []
|
||||
for path in IMAGES_ROOT.rglob("*"):
|
||||
if path.suffix not in TEMP_STAGING_SUFFIXES or not path.is_file():
|
||||
if not is_temp_staging_file(path) or not path.is_file():
|
||||
continue
|
||||
try:
|
||||
if path.stat().st_mtime < cutoff: # fresher may be an active download
|
||||
stale.append(path)
|
||||
except OSError as exc:
|
||||
log.warning("cleanup_orphaned_temp_files: %s: %s", path, exc)
|
||||
if not stale:
|
||||
return 0
|
||||
owned = _paths_with_rows([str(p) for p in stale])
|
||||
removed = 0
|
||||
for path in stale:
|
||||
if str(path) in owned:
|
||||
log.warning(
|
||||
"cleanup_orphaned_temp_files: keeping %s: a library row points at it",
|
||||
path,
|
||||
)
|
||||
continue
|
||||
try:
|
||||
if path.stat().st_mtime >= cutoff:
|
||||
continue # too fresh — may be an active download
|
||||
path.unlink()
|
||||
removed += 1
|
||||
except OSError as exc:
|
||||
@@ -715,6 +798,32 @@ def recover_stalled_download_events() -> int:
|
||||
return events_recovered
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.maintenance.date_posts_from_records",
|
||||
soft_time_limit=1500,
|
||||
time_limit=1800,
|
||||
)
|
||||
def date_posts_from_records() -> dict:
|
||||
"""Date undated native posts from the records their walk left on disk
|
||||
(#4436). Hourly and self-limiting: once every post is dated it is one
|
||||
empty query."""
|
||||
from ..services.importer import Importer
|
||||
from ..services.post_record_repair import date_posts_from_records as _repair
|
||||
from ..services.thumbnailer import Thumbnailer
|
||||
|
||||
images_root = IMAGES_ROOT
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
importer = Importer(
|
||||
session=session,
|
||||
images_root=images_root,
|
||||
import_root=images_root,
|
||||
thumbnailer=Thumbnailer(images_root=images_root),
|
||||
settings=ImportSettings.load_sync(session),
|
||||
)
|
||||
return _repair(session, importer, images_root)
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_backup_runs")
|
||||
def recover_stalled_backup_runs() -> int:
|
||||
"""Flip BackupRun rows stuck in running/restoring past the hard limit
|
||||
@@ -735,11 +844,20 @@ def recover_stalled_backup_runs() -> int:
|
||||
result = session.execute(
|
||||
update(BackupRun)
|
||||
.where(BackupRun.status.in_(["running", "restoring"]))
|
||||
# db backups/restores are fast (40-min window); images run hours (7h).
|
||||
# db backups/restores are fast (40-min window); a test restore and the
|
||||
# weekly bucket check get 7 h.
|
||||
# A media run or an off-site restore spans many chunks and hours on
|
||||
# purpose, so its window runs from its last progress, not its start.
|
||||
.where(
|
||||
or_(
|
||||
and_(BackupRun.kind == "db", BackupRun.started_at < db_cutoff),
|
||||
and_(BackupRun.kind != "db", BackupRun.started_at < slow_cutoff),
|
||||
and_(
|
||||
BackupRun.kind.in_(["media", "restore"]),
|
||||
func.coalesce(BackupRun.last_progress_at, BackupRun.started_at)
|
||||
< slow_cutoff,
|
||||
),
|
||||
and_(BackupRun.kind.not_in(["db", "media", "restore"]),
|
||||
BackupRun.started_at < slow_cutoff),
|
||||
)
|
||||
)
|
||||
.values(status="error", finished_at=now, error=msg)
|
||||
@@ -1041,8 +1159,7 @@ def cleanup_old_download_events() -> int:
|
||||
|
||||
def _backfill_wip_tier(session, tag_id, prefilter, matcher, source) -> int:
|
||||
"""One keyset-paginated pass over posts whose title matches a WIP tier, applying
|
||||
`tag_id` (stamped `source`) to their images. Shared by the hard + soft tiers
|
||||
(#1458 / #1474). Coarse `prefilter` (ILIKE superset) narrows the scan; the precise
|
||||
`tag_id` (stamped `source`) to their images (#1458). Coarse `prefilter` (ILIKE superset) narrows the scan; the precise
|
||||
`matcher` confirms. Idempotent-additive (ON CONFLICT DO NOTHING). Returns the row
|
||||
count newly applied."""
|
||||
from ..models import Post
|
||||
@@ -1082,26 +1199,17 @@ def _backfill_wip_tier(session, tag_id, prefilter, matcher, source) -> int:
|
||||
)
|
||||
def backfill_wip_title_tags() -> int:
|
||||
"""Scan EXISTING posts for WIP titles and apply the `wip` system tag to their
|
||||
images — the operator-triggered back-catalogue catch-up (task #1458 hard tier +
|
||||
#1474 soft tier). New imports are tagged live by the importer; this covers the
|
||||
existing library.
|
||||
|
||||
HARD tier ("WIP"/"work in progress") always runs (the operator triggered the
|
||||
scan); the SOFT tier (sketch/doodle, provisional source) runs only when
|
||||
wip_soft_title_tagging_enabled, AFTER hard so a title matching both keeps the
|
||||
trained hard tag (ON CONFLICT DO NOTHING). Keyset-paginated, restart-safe.
|
||||
images — the operator-triggered back-catalogue catch-up (task #1458). New
|
||||
imports are tagged live by the importer; this covers the existing library.
|
||||
Keyset-paginated, restart-safe.
|
||||
|
||||
Deliberately NOT scheduled as a beat: a periodic re-run would re-apply to matching
|
||||
posts and silently undo a manual WIP removal, so it stays an explicit operator
|
||||
action (Settings → "Scan existing posts for WIP titles"). Returns rows applied.
|
||||
"""
|
||||
from ..models import ImportSettings
|
||||
from ..services.wip_title import (
|
||||
SOFT_WIP_TITLE_SQL_PREFILTER,
|
||||
WIP_TITLE_SOFT_SOURCE,
|
||||
WIP_TITLE_SOURCE,
|
||||
WIP_TITLE_SQL_PREFILTER,
|
||||
matches_soft_wip_title,
|
||||
matches_wip_title,
|
||||
resolve_wip_tag_id,
|
||||
)
|
||||
@@ -1114,16 +1222,10 @@ def backfill_wip_title_tags() -> int:
|
||||
"backfill_wip_title_tags: no `wip` system tag present; nothing to do"
|
||||
)
|
||||
return 0
|
||||
settings = ImportSettings.load_sync(session)
|
||||
applied = _backfill_wip_tier(
|
||||
session, tag_id, WIP_TITLE_SQL_PREFILTER, matches_wip_title,
|
||||
WIP_TITLE_SOURCE,
|
||||
)
|
||||
if settings.wip_soft_title_tagging_enabled:
|
||||
applied += _backfill_wip_tier(
|
||||
session, tag_id, SOFT_WIP_TITLE_SQL_PREFILTER, matches_soft_wip_title,
|
||||
WIP_TITLE_SOFT_SOURCE,
|
||||
)
|
||||
if applied:
|
||||
log.info("backfill_wip_title_tags: applied wip to %d image(s)", applied)
|
||||
return applied
|
||||
|
||||
@@ -620,24 +620,6 @@ def scheduled_process_auto_apply() -> str:
|
||||
return f"applied={result['n_applied']} flagged={result['n_flagged']}"
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.ml.scheduled_soft_wip_conflict_audit",
|
||||
soft_time_limit=1800, time_limit=2100,
|
||||
)
|
||||
def scheduled_soft_wip_conflict_audit() -> str:
|
||||
"""Ring-loud audit over the SOFT WIP-title cohort (#1474) — flag sketch/doodle
|
||||
auto-tags that ALSO look like real content for review. No-op when there are no
|
||||
content heads; idempotent (already-flagged images skipped). Runs regardless of
|
||||
the process-sweep toggle, since soft-title tags come from the importer, not that
|
||||
sweep. Wall-clock bounded by the task time limits."""
|
||||
from ..services.ml.heads import soft_wip_conflict_audit
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
result = soft_wip_conflict_audit(session)
|
||||
return f"scanned={result['n_scanned']} flagged={result['n_flagged']}"
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.ml.prune_presentation_reviews")
|
||||
def prune_presentation_reviews() -> str:
|
||||
"""Retention (rule 89): drop RESOLVED presentation-review flags older than 30
|
||||
|
||||
@@ -175,12 +175,20 @@ def _run_probe(path_str: str) -> tuple[str, str | None]:
|
||||
return ("ok", None)
|
||||
|
||||
|
||||
def _over_cap(total) -> bool:
|
||||
return total is not None and total > MAX_ARCHIVE_UNCOMPRESSED_BYTES
|
||||
|
||||
|
||||
def _inspect_archive(path: Path):
|
||||
"""Return (total_uncompressed_bytes | None, first_bad_member | None)
|
||||
for the archive. Format detected by extension OR magic bytes (so a
|
||||
mis-named archive is still bomb-guarded + integrity-tested, matching the
|
||||
extractor's gate); raises on a structurally-broken container (caught by the
|
||||
child as a clean rejection)."""
|
||||
child as a clean rejection).
|
||||
|
||||
The size total comes from the archive's directory, so an archive over the
|
||||
cap is rejected before the integrity test decompresses anything: on a
|
||||
multi-gigabyte archive that test alone outlasts the probe's timeout."""
|
||||
from ..services.archive_extractor import detect_archive_format
|
||||
|
||||
fmt = detect_archive_format(path)
|
||||
@@ -189,12 +197,16 @@ def _inspect_archive(path: Path):
|
||||
|
||||
with zipfile.ZipFile(path) as zf:
|
||||
total = sum(zi.file_size for zi in zf.infolist())
|
||||
if _over_cap(total):
|
||||
return total, None
|
||||
return total, zf.testzip()
|
||||
if fmt == "rar":
|
||||
import rarfile
|
||||
|
||||
with rarfile.RarFile(path) as rf:
|
||||
total = sum(getattr(ri, "file_size", 0) for ri in rf.infolist())
|
||||
if _over_cap(total):
|
||||
return total, None
|
||||
rf.testrar()
|
||||
return total, None
|
||||
if fmt == "7z":
|
||||
@@ -203,6 +215,8 @@ def _inspect_archive(path: Path):
|
||||
with py7zr.SevenZipFile(path, "r") as zf:
|
||||
info = zf.archiveinfo()
|
||||
total = getattr(info, "uncompressed", None)
|
||||
if _over_cap(total):
|
||||
return total, None
|
||||
ok = zf.test() # True / None when all members pass
|
||||
return total, (None if ok in (True, None) else "7z test reported corruption")
|
||||
# Not a recognised archive — nothing to test; treat as clean.
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Downloader staging files.
|
||||
|
||||
Downloads and imports write into a staging name and `os.replace()` it into
|
||||
place when the write completes (importer, external_fetch, native_ingest_common,
|
||||
attachment_store), so a write killed midway leaves a staging file, never a
|
||||
truncated final file. Any other file is complete.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
TEMP_STAGING_SUFFIXES = (".part", ".partial")
|
||||
# megatools stages a Mega download as `.megatmp.<id>` in the target dir.
|
||||
TEMP_STAGING_PREFIXES = (".megatmp.",)
|
||||
|
||||
|
||||
def is_temp_staging_file(path: Path) -> bool:
|
||||
"""True for a downloader's staging file: one that never becomes library
|
||||
content under that name, so a stale one is safe to delete."""
|
||||
return path.suffix in TEMP_STAGING_SUFFIXES or path.name.startswith(
|
||||
TEMP_STAGING_PREFIXES
|
||||
)
|
||||
+4
-1
@@ -180,7 +180,10 @@ per `docs/process.md`'s "add deps to the image when used by >1 project".
|
||||
BUILD_REF` that every checkout in the file takes, rather than per job —
|
||||
otherwise `sign-extension` would derive dev's extension version while
|
||||
`build-web` bundled main's, and the release download would 404 on a version
|
||||
that exists perfectly well. Every job then ASSERTS its checkout is `main`
|
||||
that exists perfectly well. On every other trigger `BUILD_REF` is the
|
||||
triggering COMMIT (`github.sha`), not the branch: a branch is re-resolved
|
||||
per job, so a push landing mid-run used to move the publishing jobs onto a
|
||||
commit the run's lanes never tested (run 7499, #4427). Every job then ASSERTS its checkout is `main`
|
||||
before doing anything, because `env` inside `with:` is not a context this
|
||||
runner is known to evaluate — if it silently resolved to empty, checkout
|
||||
would fall back to the triggering ref and the refresh would publish dev's
|
||||
|
||||
@@ -1,11 +1,6 @@
|
||||
/**
|
||||
* Background script — message router + Discord token capture (webRequest).
|
||||
* Direct port of GS background.js; api.js client points at FC instead of GS.
|
||||
*
|
||||
* pixiv's PKCE OAuth flow lived here until FC retired pixiv (milestone #406).
|
||||
* It was removed together with pixiv's host permissions rather than left
|
||||
* behind: a webRequest listener on a host the manifest no longer grants is at
|
||||
* best dead and at worst a startup failure for the whole background script.
|
||||
*/
|
||||
|
||||
let discordToken = null;
|
||||
@@ -17,19 +12,9 @@ async function ensureInitialized() {
|
||||
if (initialized) return;
|
||||
await api.init();
|
||||
await loadDiscordToken();
|
||||
await forgetRetiredPixivToken();
|
||||
initialized = true;
|
||||
}
|
||||
|
||||
// A browser that authenticated pixiv before the retirement still holds a live
|
||||
// OAuth refresh token in extension storage. Nothing reads it any more, and a
|
||||
// credential for a service FC no longer uses is a liability with no benefit
|
||||
// (the same reasoning as the server-side cleanup, issue #3980). Removing keys
|
||||
// that are absent is a no-op, so this is safe on every startup.
|
||||
async function forgetRetiredPixivToken() {
|
||||
await browser.storage.local.remove(['pixivRefreshToken', 'pixivTokenCapturedAt']);
|
||||
}
|
||||
|
||||
browser.runtime.onInstalled.addListener(() => ensureInitialized());
|
||||
browser.runtime.onStartup.addListener(() => ensureInitialized());
|
||||
ensureInitialized().catch(e => console.error('init failed:', e));
|
||||
@@ -88,7 +73,12 @@ async function checkForUpdateInfo() {
|
||||
currentVersion,
|
||||
latestVersion,
|
||||
channel,
|
||||
xpiUrl: info && info.latest_url ? `${base}${info.latest_url}` : null,
|
||||
// Where the Update button sends the operator: FC's own install card, not
|
||||
// the XPI. Firefox refuses an add-on install whose navigation an extension
|
||||
// started (tabs.create on the .xpi dies with NS_ERROR_FAILURE — operator-
|
||||
// flagged 2026-09-25); it accepts one from a user click on a web page,
|
||||
// which is exactly what the card's Install button is.
|
||||
installPageUrl: base ? `${base}/subscriptions?tab=settings` : null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -270,11 +260,27 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
||||
return await api.quickAddSource(msg.url, {
|
||||
artistId: msg.artistId ?? null,
|
||||
artistName: msg.artistName ?? null,
|
||||
usePlatformName: msg.usePlatformName === true,
|
||||
discordAuthors: Array.isArray(msg.discordAuthors) ? msg.discordAuthors : null,
|
||||
});
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
case 'DISCORD_POSTERS':
|
||||
try {
|
||||
return await api.getDiscordPosters(msg.url);
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
case 'SET_DISCORD_POSTERS':
|
||||
try {
|
||||
return await api.setDiscordPosters(msg.url, msg.authors || []);
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
case 'SEARCH_ARTISTS':
|
||||
try {
|
||||
return { artists: await api.searchArtists(msg.q || '') };
|
||||
@@ -284,7 +290,7 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
||||
|
||||
case 'PROBE_SOURCE':
|
||||
try {
|
||||
return await api.probeSource(msg.url);
|
||||
return await api.probeSource(msg.url, { names: msg.names === true });
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
@@ -82,3 +82,25 @@
|
||||
}
|
||||
.fc-panel__btn--primary { border-color: rgb(244, 186, 122); background: rgb(244, 186, 122); color: rgb(20, 23, 26); }
|
||||
.fc-panel__btn:disabled { opacity: 0.5; cursor: default; }
|
||||
|
||||
/* Artist autocomplete — the list sits under the field, inside the panel. */
|
||||
.fc-panel__combo { position: relative; }
|
||||
.fc-panel__results {
|
||||
margin-top: 4px; border-radius: 6px;
|
||||
background: rgb(12, 14, 16);
|
||||
}
|
||||
.fc-panel__results:empty { display: none; }
|
||||
.fc-panel__results:not(:empty) { border: 1px solid rgb(70, 74, 80); padding: 3px; }
|
||||
.fc-panel__result { display: flex; justify-content: space-between; align-items: center; width: 100%; box-sizing: border-box; }
|
||||
.fc-panel__result--active { background: rgb(52, 44, 32); outline: 1px solid rgb(244, 186, 122); }
|
||||
.fc-panel__result--new { color: rgb(244, 186, 122); }
|
||||
.fc-panel__result-tag { font-size: 11px; color: rgb(140, 220, 160); }
|
||||
.fc-panel__empty { padding: 6px 9px; font-size: 12px; color: rgb(170, 166, 156); }
|
||||
.fc-panel__hint--match { color: rgb(140, 220, 160); }
|
||||
/* The panel's own [hidden] — its rows are display:flex, which beats the UA's. */
|
||||
.fc-panel [hidden] { display: none !important; }
|
||||
.fc-panel__rename { margin-top: 8px; font-size: 13px; }
|
||||
/* Poster picker (#4491): who posts in the channel, the likely creator ticked. */
|
||||
.fc-panel__posters { max-height: 220px; overflow-y: auto; }
|
||||
.fc-panel__poster { flex-wrap: wrap; }
|
||||
.fc-panel__poster-meta { flex-basis: 100%; padding-left: 21px; font-size: 11px; color: rgb(170, 166, 156); }
|
||||
|
||||
@@ -81,16 +81,24 @@
|
||||
const probe = currentProbe;
|
||||
|
||||
if (probe?.state === 'source_match') {
|
||||
// A followed Discord channel opens its poster list (#4491); the artist
|
||||
// page is one click further, in that panel.
|
||||
if (probe.platform === 'discord' && probe.discord?.channel_id) {
|
||||
if (document.getElementById('fc-add-panel')) closePanel();
|
||||
else openPostersPanel(btn, probe);
|
||||
return;
|
||||
}
|
||||
await openArtist(btn, probe.artist?.slug);
|
||||
return;
|
||||
}
|
||||
// A Discord URL names a channel, not a creator — ask which artist.
|
||||
if (probe?.platform === 'discord') {
|
||||
if (document.getElementById('fc-discord-panel')) closePanel();
|
||||
else openDiscordPanel(probe);
|
||||
// Every add goes through the panel, so the operator can match the page to
|
||||
// an artist FabledCurator already has (the same creator is often spelled
|
||||
// differently per platform) instead of minting a duplicate.
|
||||
if (document.getElementById('fc-add-panel')) {
|
||||
closePanel();
|
||||
return;
|
||||
}
|
||||
await add(btn, { url: window.location.href });
|
||||
await openAddPanel(btn, probe);
|
||||
}
|
||||
|
||||
async function openArtist(btn, slug) {
|
||||
@@ -121,7 +129,8 @@
|
||||
return false;
|
||||
}
|
||||
const verb = r.created_source ? 'Added to' : 'Already a source for';
|
||||
showToast(`${verb} ${r.artist?.name || 'artist'} (${r.source?.platform || ''})`, 'success');
|
||||
const renamed = r.renamed_from ? ` — renamed from “${r.renamed_from}”` : '';
|
||||
showToast(`${verb} ${r.artist?.name || 'artist'} (${r.source?.platform || ''})${renamed}`, 'success');
|
||||
// Re-probe so the chip flips green without waiting for a navigation.
|
||||
evaluate();
|
||||
return true;
|
||||
@@ -134,10 +143,13 @@
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Discord Add panel ----
|
||||
// Where: this channel or the whole server. Who: the suggested artist, one
|
||||
// found by search, or a new one by name. Built with createElement only —
|
||||
// server, channel and artist names are other people's text.
|
||||
// ---- Add panel ----
|
||||
// Who: the suggested artist, one found by search, or a new one by name —
|
||||
// on every platform. Where (Discord only): this channel or the whole
|
||||
// server. On Patreon, joining an artist known by another name offers the
|
||||
// Patreon name, which the operator treats as canon. Built with
|
||||
// createElement only — server, channel and artist names are other people's
|
||||
// text.
|
||||
|
||||
function el(tag, props = {}, children = []) {
|
||||
const node = document.createElement(tag);
|
||||
@@ -150,14 +162,151 @@
|
||||
}
|
||||
|
||||
function closePanel() {
|
||||
document.getElementById('fc-discord-panel')?.remove();
|
||||
document.getElementById('fc-add-panel')?.remove();
|
||||
}
|
||||
|
||||
function openDiscordPanel(probe) {
|
||||
// ---- Poster picker (#4491) ----
|
||||
// Who posted in this channel lately, read by FabledCurator with its Discord
|
||||
// token, the creator the server is named for (or owned by) ticked. The
|
||||
// source keeps their ids: a username can change on a whim, an id can't, so
|
||||
// the list never quietly stops matching. Names are other people's text —
|
||||
// createElement only.
|
||||
|
||||
function posterSection() {
|
||||
const list = el('div', { class: 'fc-panel__posters' }, [
|
||||
el('div', { class: 'fc-panel__hint', text: 'Reading who posts here…' }),
|
||||
]);
|
||||
const node = el('div', {}, [
|
||||
el('div', { class: 'fc-panel__label', text: 'Only posts by' }),
|
||||
list,
|
||||
]);
|
||||
const state = { data: null, picked: new Set() };
|
||||
|
||||
function render() {
|
||||
const posters = withSelected(state.data);
|
||||
if (!posters.length) {
|
||||
list.replaceChildren(el('div', { class: 'fc-panel__hint', text: 'No one has posted here lately.' }));
|
||||
return;
|
||||
}
|
||||
const rows = posters.map((p) => {
|
||||
const id = String(p.id);
|
||||
const box = el('input', { type: 'checkbox', checked: state.picked.has(id) });
|
||||
box.addEventListener('change', () => {
|
||||
if (box.checked) state.picked.add(id);
|
||||
else state.picked.delete(id);
|
||||
hint.textContent = pickHint();
|
||||
});
|
||||
const bits = [`${p.images} image${p.images === 1 ? '' : 's'}`, `${p.messages} message${p.messages === 1 ? '' : 's'}`];
|
||||
if (p.reasons?.length) bits.unshift(p.reasons.join(', '));
|
||||
return el('label', { class: 'fc-panel__radio fc-panel__poster' }, [
|
||||
box,
|
||||
el('span', { text: posterLabel(p) }),
|
||||
el('span', { class: 'fc-panel__poster-meta', text: bits.join(' · ') }),
|
||||
]);
|
||||
});
|
||||
const hint = el('div', { class: 'fc-panel__hint', text: pickHint() });
|
||||
list.replaceChildren(...rows, hint);
|
||||
}
|
||||
|
||||
function pickHint() {
|
||||
return state.picked.size
|
||||
? 'Only the ticked people’s posts are taken.'
|
||||
: 'Nothing ticked: everyone’s posts are taken.';
|
||||
}
|
||||
|
||||
async function load() {
|
||||
let r;
|
||||
try {
|
||||
r = await browser.runtime.sendMessage({ type: 'DISCORD_POSTERS', url: window.location.href });
|
||||
} catch (e) {
|
||||
r = { error: e.message };
|
||||
}
|
||||
if (r?.error) {
|
||||
list.replaceChildren(el('div', { class: 'fc-panel__hint', text: `Couldn’t list posters — ${r.error}` }));
|
||||
return;
|
||||
}
|
||||
state.data = r;
|
||||
state.picked = initialPosterPick(r);
|
||||
render();
|
||||
}
|
||||
|
||||
// null until the list has loaded: an add made before then leaves the
|
||||
// source's list as it is instead of clearing it.
|
||||
const request = () => (state.data ? postersRequest(state.data.posters, state.picked) : null);
|
||||
return { node, load, request };
|
||||
}
|
||||
|
||||
function openPostersPanel(btn, probe) {
|
||||
closePanel();
|
||||
const d = probe.discord || {};
|
||||
const choice = discordPanelDefaults(probe);
|
||||
let searchSeq = 0;
|
||||
const posters = posterSection();
|
||||
const saveBtn = el('button', { class: 'fc-panel__btn fc-panel__btn--primary', text: 'Save' });
|
||||
const artistBtn = el('button', { class: 'fc-panel__btn', text: `Open ${probe.artist?.name || 'artist'}` });
|
||||
const cancelBtn = el('button', { class: 'fc-panel__btn', text: 'Close' });
|
||||
const panel = el('div', { id: 'fc-add-panel', class: 'fc-panel' }, [
|
||||
el('div', { class: 'fc-panel__title', text: 'Discord source' }),
|
||||
el('div', { class: 'fc-panel__sub', text: d.channel_id ? channelLabel(d) : serverLabel(d) }),
|
||||
posters.node,
|
||||
el('div', { class: 'fc-panel__actions' }, [artistBtn, cancelBtn, saveBtn]),
|
||||
]);
|
||||
panel.addEventListener('keydown', (e) => {
|
||||
e.stopPropagation();
|
||||
if (e.key === 'Escape') closePanel();
|
||||
});
|
||||
cancelBtn.addEventListener('click', closePanel);
|
||||
artistBtn.addEventListener('click', () => { closePanel(); openArtist(btn, probe.artist?.slug); });
|
||||
saveBtn.addEventListener('click', async () => {
|
||||
const authors = posters.request();
|
||||
if (!authors) return;
|
||||
saveBtn.disabled = true;
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({
|
||||
type: 'SET_DISCORD_POSTERS', url: window.location.href, authors,
|
||||
});
|
||||
if (r?.error) {
|
||||
showToast(`Error: ${r.error}`, 'error');
|
||||
return;
|
||||
}
|
||||
showToast(authors.length ? `Only posts by ${authors.map((a) => a.name || a.id).join(', ')}` : 'Taking everyone’s posts', 'success');
|
||||
closePanel();
|
||||
} catch (e) {
|
||||
showToast(`Error: ${e.message}`, 'error');
|
||||
} finally {
|
||||
saveBtn.disabled = false;
|
||||
}
|
||||
});
|
||||
document.body.appendChild(panel);
|
||||
posters.load();
|
||||
}
|
||||
|
||||
async function openAddPanel(btn, probe) {
|
||||
closePanel();
|
||||
const platformName = PLATFORMS[probe.platform]?.name || probe.platform;
|
||||
// Discord's probe already carries its names. Patreon/SubscribeStar read the
|
||||
// creator's display name only now, when the panel needs it — a request to
|
||||
// the platform the chip's own probe deliberately doesn't make.
|
||||
if (probe.platform !== 'discord') {
|
||||
const original = btn.textContent;
|
||||
btn.disabled = true;
|
||||
btn.textContent = `Reading the ${platformName} name…`;
|
||||
try {
|
||||
const named = await browser.runtime.sendMessage({
|
||||
type: 'PROBE_SOURCE', url: window.location.href, names: true,
|
||||
});
|
||||
if (named && !named.error) probe = named;
|
||||
} catch { /* fall back to the chip's probe: the URL handle */ }
|
||||
btn.disabled = false;
|
||||
btn.textContent = original;
|
||||
if (probe.state === 'source_match') {
|
||||
renderButton(probe);
|
||||
return;
|
||||
}
|
||||
}
|
||||
const d = probe.discord || {};
|
||||
const discord = probe.platform === 'discord';
|
||||
const choice = panelDefaults(probe, window.location.href);
|
||||
// #4491: on a channel page, pick whose posts the new source takes.
|
||||
const posters = discord && d.channel_id ? posterSection() : null;
|
||||
|
||||
const scopeRow = (value, label, disabled) => {
|
||||
const input = el('input', {
|
||||
@@ -168,58 +317,127 @@
|
||||
return el('label', { class: 'fc-panel__radio' }, [input, el('span', { text: label })]);
|
||||
};
|
||||
|
||||
// The artist field is an autocomplete over FC's artists: it searches as
|
||||
// soon as the panel opens (with the prefilled name) and on every keystroke,
|
||||
// lists the matches under the field, fills in the rest of the top match as
|
||||
// you type (Tab or Enter accepts it, typing on replaces it), and picks an
|
||||
// artist whose name IS the text, spacing and case aside, without being
|
||||
// asked. ↑/↓ walk the list; the last row creates a new artist instead.
|
||||
const nameInput = el('input', {
|
||||
type: 'text', class: 'fc-panel__input', value: choice.artistName,
|
||||
placeholder: 'Artist name — search or type a new one',
|
||||
placeholder: 'Search artists or type a new name',
|
||||
autocomplete: 'off', spellcheck: false,
|
||||
role: 'combobox',
|
||||
});
|
||||
const results = el('div', { class: 'fc-panel__results' });
|
||||
nameInput.setAttribute('aria-autocomplete', 'both');
|
||||
nameInput.setAttribute('aria-expanded', 'false');
|
||||
const results = el('div', { class: 'fc-panel__results', role: 'listbox' });
|
||||
const hint = el('div', { class: 'fc-panel__hint' });
|
||||
const addBtn = el('button', { class: 'fc-panel__btn fc-panel__btn--primary', text: 'Add' });
|
||||
const cancelBtn = el('button', { class: 'fc-panel__btn', text: 'Cancel' });
|
||||
|
||||
const panel = el('div', { id: 'fc-discord-panel', class: 'fc-panel' }, [
|
||||
el('div', { class: 'fc-panel__title', text: 'Add Discord source' }),
|
||||
el('div', { class: 'fc-panel__sub', text: serverLabel(d) }),
|
||||
el('div', { class: 'fc-panel__label', text: 'Follow' }),
|
||||
scopeRow('channel', d.channel_id ? channelLabel(d) : 'this channel', !d.channel_id),
|
||||
scopeRow('server', `Every channel in ${serverLabel(d)}`, false),
|
||||
// Patreon is canon: joining an artist known by another name takes the
|
||||
// Patreon name unless this is unticked. Shown only when it would rename.
|
||||
const renameBox = el('input', { type: 'checkbox', checked: choice.adoptPlatformName });
|
||||
const renameText = el('span');
|
||||
const renameRow = el('label', { class: 'fc-panel__radio fc-panel__rename' }, [renameBox, renameText]);
|
||||
renameBox.addEventListener('change', () => { choice.adoptPlatformName = renameBox.checked; refresh(); });
|
||||
|
||||
const sub = discord
|
||||
? serverLabel(d)
|
||||
: [probe.display_name, probe.slug].filter(Boolean).filter((v, i, a) => a.indexOf(v) === i).join(' · ');
|
||||
const panel = el('div', { id: 'fc-add-panel', class: 'fc-panel' }, [
|
||||
el('div', { class: 'fc-panel__title', text: `Add ${platformName} source` }),
|
||||
el('div', { class: 'fc-panel__sub', text: sub }),
|
||||
...(discord ? [
|
||||
el('div', { class: 'fc-panel__label', text: 'Follow' }),
|
||||
scopeRow('channel', d.channel_id ? channelLabel(d) : 'this channel', !d.channel_id),
|
||||
scopeRow('server', `Every channel in ${serverLabel(d)}`, false),
|
||||
] : []),
|
||||
el('div', { class: 'fc-panel__label', text: 'Artist' }),
|
||||
nameInput,
|
||||
results,
|
||||
el('div', { class: 'fc-panel__combo' }, [nameInput, results]),
|
||||
renameRow,
|
||||
...(posters ? [posters.node] : []),
|
||||
hint,
|
||||
el('div', { class: 'fc-panel__actions' }, [cancelBtn, addBtn]),
|
||||
]);
|
||||
|
||||
// Search state: the rows on screen, which one ↑/↓ has highlighted (-1 =
|
||||
// none), and whether the list is open.
|
||||
let rows = [];
|
||||
let active = -1;
|
||||
let listOpen = false;
|
||||
|
||||
function refresh() {
|
||||
const req = discordAddRequest(choice);
|
||||
const req = addRequest(choice);
|
||||
addBtn.disabled = !req;
|
||||
if (!req) hint.textContent = 'Pick an artist or type a name.';
|
||||
else if (req.artistId != null) hint.textContent = `Connects to ${choice.artist.name} in FabledCurator.`;
|
||||
else hint.textContent = `Adds to “${req.artistName}” — created if FabledCurator has no artist by that name.`;
|
||||
else if (req.artistId != null) hint.textContent = `✓ Connects to ${choice.artist.name}, already in FabledCurator.`;
|
||||
else if (req.artistName) hint.textContent = `Creates a new artist “${req.artistName}”.`;
|
||||
else hint.textContent = `Creates a new artist, named from the ${platformName} page.`;
|
||||
hint.classList.toggle('fc-panel__hint--match', !!req && req.artistId != null);
|
||||
const offer = renameOffer(choice);
|
||||
renameRow.hidden = !offer;
|
||||
if (offer) renameText.textContent = `Rename “${offer.from}” to the Patreon name “${offer.to}”`;
|
||||
}
|
||||
|
||||
function showResults(rows) {
|
||||
results.replaceChildren(...rows.map((a) => {
|
||||
const row = el('button', { class: 'fc-panel__result', text: a.name });
|
||||
row.addEventListener('click', () => {
|
||||
choice.artist = { id: a.id, name: a.name };
|
||||
choice.artistName = a.name;
|
||||
nameInput.value = a.name;
|
||||
results.replaceChildren();
|
||||
refresh();
|
||||
});
|
||||
function pick(artist) {
|
||||
choice.artist = artist ? { id: artist.id, name: artist.name } : null;
|
||||
if (artist) {
|
||||
choice.artistName = artist.name;
|
||||
nameInput.value = artist.name;
|
||||
}
|
||||
closeList();
|
||||
refresh();
|
||||
}
|
||||
|
||||
function closeList() {
|
||||
listOpen = false;
|
||||
active = -1;
|
||||
results.replaceChildren();
|
||||
nameInput.setAttribute('aria-expanded', 'false');
|
||||
}
|
||||
|
||||
function renderList() {
|
||||
const typed = nameInput.value.trim();
|
||||
const exact = exactArtistMatch(typed, rows);
|
||||
const items = rows.map((a, i) => {
|
||||
const row = el('button', { type: 'button', class: 'fc-panel__result', role: 'option' }, [
|
||||
el('span', { text: a.name }),
|
||||
]);
|
||||
if (choice.artist && choice.artist.id === a.id) {
|
||||
row.appendChild(el('span', { class: 'fc-panel__result-tag', text: 'selected' }));
|
||||
}
|
||||
row.classList.toggle('fc-panel__result--active', i === active);
|
||||
// mousedown, not click: it fires before the input's blur closes the list.
|
||||
row.addEventListener('mousedown', (e) => { e.preventDefault(); pick(a); });
|
||||
return row;
|
||||
}));
|
||||
});
|
||||
if (typed && !exact) {
|
||||
const i = rows.length;
|
||||
const create = el('button', { type: 'button', class: 'fc-panel__result fc-panel__result--new', role: 'option',
|
||||
text: `+ New artist “${typed}”` });
|
||||
create.classList.toggle('fc-panel__result--active', i === active);
|
||||
create.addEventListener('mousedown', (e) => { e.preventDefault(); pick(null); choice.artistName = typed; refresh(); });
|
||||
items.push(create);
|
||||
}
|
||||
if (typed && !rows.length) {
|
||||
items.unshift(el('div', { class: 'fc-panel__empty', text: 'No FabledCurator artist matches.' }));
|
||||
}
|
||||
results.replaceChildren(...items);
|
||||
listOpen = items.length > 0;
|
||||
nameInput.setAttribute('aria-expanded', String(listOpen));
|
||||
results.querySelector('.fc-panel__result--active')?.scrollIntoView({ block: 'nearest' });
|
||||
}
|
||||
|
||||
let debounce = null;
|
||||
nameInput.addEventListener('input', () => {
|
||||
choice.artistName = nameInput.value;
|
||||
refresh();
|
||||
let searchSeq = 0;
|
||||
// `autofill` is false for deletions: filling the name back in as you
|
||||
// backspace would make it impossible to delete.
|
||||
function search(autofill) {
|
||||
clearTimeout(debounce);
|
||||
const q = nameInput.value.trim();
|
||||
if (!q) { results.replaceChildren(); return; }
|
||||
if (!q) { rows = []; closeList(); return; }
|
||||
debounce = setTimeout(async () => {
|
||||
const mine = ++searchSeq;
|
||||
let r;
|
||||
@@ -228,21 +446,78 @@
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (mine !== searchSeq || r?.error) return;
|
||||
showResults(r.artists || []);
|
||||
}, 200);
|
||||
// Stale: a later keystroke has its own search coming.
|
||||
if (mine !== searchSeq || r?.error || nameInput.value.trim() !== q) return;
|
||||
rows = r.artists || [];
|
||||
active = -1;
|
||||
const exact = exactArtistMatch(q, rows);
|
||||
if (exact && !choice.artist) {
|
||||
choice.artist = { id: exact.id, name: exact.name };
|
||||
} else if (autofill && document.activeElement === nameInput) {
|
||||
const hit = inlineCompletion(nameInput.value, rows);
|
||||
const caret = nameInput.value.length;
|
||||
if (hit && nameInput.selectionStart === caret) {
|
||||
nameInput.value = nameInput.value + hit.name.slice(caret);
|
||||
nameInput.setSelectionRange(caret, hit.name.length);
|
||||
choice.artist = { id: hit.id, name: hit.name };
|
||||
choice.artistName = hit.name;
|
||||
}
|
||||
}
|
||||
renderList();
|
||||
refresh();
|
||||
}, 150);
|
||||
}
|
||||
|
||||
nameInput.addEventListener('input', (e) => {
|
||||
choice.artistName = nameInput.value;
|
||||
// Any edit un-picks: the artist is whatever the field now says.
|
||||
choice.artist = null;
|
||||
refresh();
|
||||
search(!String(e.inputType || '').startsWith('delete'));
|
||||
});
|
||||
nameInput.addEventListener('focus', () => { if (rows.length) renderList(); });
|
||||
nameInput.addEventListener('blur', () => closeList());
|
||||
|
||||
// Keep Discord's global shortcuts from eating keystrokes meant for us.
|
||||
panel.addEventListener('keydown', (e) => {
|
||||
e.stopPropagation();
|
||||
if (e.key === 'Escape') closePanel();
|
||||
const count = results.querySelectorAll('.fc-panel__result').length;
|
||||
if (e.target === nameInput && listOpen && count) {
|
||||
if (e.key === 'ArrowDown' || e.key === 'ArrowUp') {
|
||||
e.preventDefault();
|
||||
active = e.key === 'ArrowDown' ? (active + 1) % count : (active <= 0 ? count - 1 : active - 1);
|
||||
renderList();
|
||||
return;
|
||||
}
|
||||
if ((e.key === 'Enter' || e.key === 'Tab') && active >= 0) {
|
||||
e.preventDefault();
|
||||
results.querySelectorAll('.fc-panel__result')[active]
|
||||
.dispatchEvent(new MouseEvent('mousedown', { cancelable: true }));
|
||||
return;
|
||||
}
|
||||
if ((e.key === 'Tab' || e.key === 'Enter') && choice.artist
|
||||
&& nameInput.selectionEnd > nameInput.selectionStart) {
|
||||
// Accept the inline autofill — Enter too, but only to accept: the
|
||||
// add itself takes a second Enter, once the hint names the artist.
|
||||
e.preventDefault();
|
||||
pick(choice.artist);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (e.key === 'Escape') {
|
||||
if (listOpen) closeList();
|
||||
else closePanel();
|
||||
return;
|
||||
}
|
||||
if (e.key === 'Enter' && e.target === nameInput && !addBtn.disabled) addBtn.click();
|
||||
});
|
||||
|
||||
cancelBtn.addEventListener('click', closePanel);
|
||||
addBtn.addEventListener('click', async () => {
|
||||
const req = discordAddRequest(choice);
|
||||
const req = addRequest(choice);
|
||||
if (!req) return;
|
||||
const authors = posters?.request();
|
||||
if (authors) req.discordAuthors = authors;
|
||||
const btn = document.getElementById('fc-add-source-btn');
|
||||
addBtn.disabled = true;
|
||||
const ok = await add(btn || addBtn, req);
|
||||
@@ -253,6 +528,11 @@
|
||||
document.body.appendChild(panel);
|
||||
refresh();
|
||||
nameInput.focus();
|
||||
posters?.load();
|
||||
nameInput.select();
|
||||
// Search what the field opens with — the server's name, usually — so an
|
||||
// artist it already matches is picked before the operator types anything.
|
||||
if (!choice.artist && nameInput.value.trim()) search(false);
|
||||
}
|
||||
|
||||
function showToast(text, kind) {
|
||||
|
||||
+23
-4
@@ -104,18 +104,37 @@ class FabledCuratorAPI {
|
||||
// artistId connects the source to an existing artist, artistName to the
|
||||
// artist of that name (created if new); with neither the server derives the
|
||||
// artist from the URL. A Discord channel always sends one.
|
||||
quickAddSource(url, { artistId = null, artistName = null } = {}) {
|
||||
// usePlatformName: a Patreon source joining an existing artist renames it
|
||||
// to the Patreon display name (Patreon is canon; name only, never the slug).
|
||||
// discordAuthors ([{id, name}], #4491): a Discord source's poster list, set
|
||||
// in the same step.
|
||||
quickAddSource(url, { artistId = null, artistName = null, usePlatformName = false, discordAuthors = null } = {}) {
|
||||
const body = { url };
|
||||
if (discordAuthors) body.discord_authors = discordAuthors;
|
||||
if (artistId != null) body.artist_id = artistId;
|
||||
else if (artistName) body.artist_name = artistName;
|
||||
if (usePlatformName) body.use_platform_name = true;
|
||||
return this.request('POST', '/extension/quick-add-source', body);
|
||||
}
|
||||
probeSource(url) {
|
||||
probeSource(url, { names = false } = {}) {
|
||||
// Read-only existence check. Drives the content-script chip's
|
||||
// color/copy BEFORE the operator clicks Add.
|
||||
const qs = new URLSearchParams({ url }).toString();
|
||||
// color/copy BEFORE the operator clicks Add. `names` also reads the
|
||||
// creator's display name from the platform — the Add panel asks for it,
|
||||
// the chip doesn't, so a plain page view never costs a platform request.
|
||||
const params = { url };
|
||||
if (names) params.names = '1';
|
||||
const qs = new URLSearchParams(params).toString();
|
||||
return this.request('GET', `/extension/probe?${qs}`);
|
||||
}
|
||||
// #4491: who posts in the Discord channel `url` shows, lately, with the
|
||||
// likely creator marked; and setting the list on the source that follows it.
|
||||
getDiscordPosters(url) {
|
||||
const qs = new URLSearchParams({ url }).toString();
|
||||
return this.request('GET', `/extension/discord/posters?${qs}`);
|
||||
}
|
||||
setDiscordPosters(url, authors) {
|
||||
return this.request('POST', '/extension/discord/posters', { url, authors });
|
||||
}
|
||||
// Latest published extension version on this instance — drives the in-app
|
||||
// update prompt. Public endpoint (no key needed, but request() sends it
|
||||
// harmlessly). Returns {version, xpi_url, latest_url, sha256}.
|
||||
|
||||
+126
-14
@@ -1,7 +1,7 @@
|
||||
/**
|
||||
* The content script's decisions, kept apart from its DOM so the specs can
|
||||
* load them (test/chip.spec.js): which state the chip shows, what it says,
|
||||
* and what the Discord Add panel starts out proposing.
|
||||
* and what the Add panel starts out proposing and finally sends.
|
||||
*
|
||||
* `probe` is /api/extension/probe's answer; `platformName` is the display
|
||||
* name (PLATFORMS[key].name), passed in so this file needs no other lib.
|
||||
@@ -46,35 +46,147 @@ function serverLabel(d) {
|
||||
}
|
||||
|
||||
/**
|
||||
* What the Discord Add panel opens with. The channel is the default scope
|
||||
* when there is one: a server source walks every channel the token can read,
|
||||
* which is rarely what a single art channel wants. The artist is the probe's
|
||||
* suggestion (the owner of another source on this server), else a new artist
|
||||
* named after the server.
|
||||
* What the Add panel opens with, on any platform.
|
||||
*
|
||||
* Discord: the channel is the default scope when there is one — a server
|
||||
* source walks every channel the token can read, which is rarely what a
|
||||
* single art channel wants. The artist is the probe's suggestion (the owner
|
||||
* of another source on this server), else a new one named after the server.
|
||||
*
|
||||
* Patreon / SubscribeStar: the page URL is the source. The artist is the one
|
||||
* whose slug the URL already names (artist_match), else a new one under the
|
||||
* creator's display name — `probe.display_name`, from the probe the panel
|
||||
* makes with names=1 — falling back to the URL handle, which the server
|
||||
* resolves on its own when it is left untouched (`nameIsHandle`).
|
||||
*/
|
||||
function discordPanelDefaults(probe) {
|
||||
function panelDefaults(probe, pageUrl) {
|
||||
const d = probe?.discord || {};
|
||||
const suggested = probe?.state === 'artist_match' && probe.artist ? probe.artist : null;
|
||||
const discord = probe?.platform === 'discord';
|
||||
const shown = probe?.display_name || null;
|
||||
let artistName = '';
|
||||
if (suggested) artistName = suggested.name;
|
||||
else if (discord) artistName = d.server_name || '';
|
||||
else artistName = shown || probe?.slug || '';
|
||||
return {
|
||||
scope: d.channel_id ? 'channel' : 'server',
|
||||
platform: probe?.platform || null,
|
||||
scope: discord ? (d.channel_id ? 'channel' : 'server') : 'page',
|
||||
pageUrl: pageUrl || null,
|
||||
channelUrl: d.channel_url || null,
|
||||
serverUrl: d.server_url || null,
|
||||
artist: suggested ? { id: suggested.id, name: suggested.name } : null,
|
||||
artistName: suggested ? suggested.name : (d.server_name || ''),
|
||||
artistName,
|
||||
nameIsHandle: !discord && !suggested && !shown,
|
||||
handle: probe?.slug || '',
|
||||
displayName: shown,
|
||||
// Patreon is canon: joining an artist known by another name takes the
|
||||
// Patreon name, unless the operator unticks it.
|
||||
adoptPlatformName: true,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The rename the panel offers, or null: only on Patreon (the canon name),
|
||||
* only when joining an existing artist, only with a name actually read from
|
||||
* Patreon, and only when it differs from what the artist is called now.
|
||||
*/
|
||||
function renameOffer(choice) {
|
||||
if (choice.platform !== 'patreon' || !choice.artist || !choice.displayName) return null;
|
||||
if (choice.artist.name === choice.displayName) return null;
|
||||
return { from: choice.artist.name, to: choice.displayName };
|
||||
}
|
||||
|
||||
/**
|
||||
* The quick-add body for the panel's current choice. A picked artist goes by
|
||||
* id — names can collide once slugified — and a typed name creates (or
|
||||
* finds) that artist. null when there is nothing valid to send.
|
||||
* finds) that artist. The URL handle left untouched sends no name, so the
|
||||
* server resolves the display name itself. null when there is nothing valid.
|
||||
*/
|
||||
function discordAddRequest(choice) {
|
||||
const url = choice.scope === 'server' ? choice.serverUrl : choice.channelUrl;
|
||||
function addRequest(choice) {
|
||||
let url = choice.pageUrl;
|
||||
if (choice.scope === 'server') url = choice.serverUrl;
|
||||
else if (choice.scope === 'channel') url = choice.channelUrl;
|
||||
if (!url) return null;
|
||||
if (choice.artist && choice.artist.id != null && choice.artist.name === choice.artistName) {
|
||||
return { url, artistId: choice.artist.id };
|
||||
const req = { url, artistId: choice.artist.id };
|
||||
if (choice.adoptPlatformName && renameOffer(choice)) req.usePlatformName = true;
|
||||
return req;
|
||||
}
|
||||
const name = (choice.artistName || '').trim();
|
||||
return name ? { url, artistName: name } : null;
|
||||
if (!name) return null;
|
||||
if (choice.nameIsHandle && name === choice.handle) return { url };
|
||||
return { url, artistName: name };
|
||||
}
|
||||
|
||||
/**
|
||||
* An artist name reduced to what identifies it: lowercase letters and digits
|
||||
* of any script, nothing else — so "Tamada Heijun", "tamada_heijun" and
|
||||
* "TamadaHeijun" are one name. Mirrors the server's autocomplete (#429).
|
||||
*/
|
||||
function squashName(name) {
|
||||
return String(name || '').toLowerCase().replace(/[^\p{L}\p{N}]/gu, '');
|
||||
}
|
||||
|
||||
/** The search result that IS the query, spacing aside, else null. */
|
||||
function exactArtistMatch(query, results) {
|
||||
const q = squashName(query);
|
||||
if (!q) return null;
|
||||
return (results || []).find((a) => squashName(a.name) === q) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inline autofill: the first result whose name extends what was typed
|
||||
* (case-insensitive), so the panel can fill in the rest and select it —
|
||||
* typing on overwrites it, Tab or Enter accepts it. null when none does.
|
||||
*/
|
||||
function inlineCompletion(typed, results) {
|
||||
const t = String(typed || '').toLowerCase();
|
||||
if (!t) return null;
|
||||
return (results || []).find((a) => a.name.length > t.length && a.name.toLowerCase().startsWith(t)) || null;
|
||||
}
|
||||
|
||||
// ---- Discord poster picker (#4491) ----
|
||||
// /api/extension/discord/posters lists who posted in the channel lately, with
|
||||
// the creator the server is named for (or owned by) marked `suggested`. The
|
||||
// picker ticks posters; the source keeps their ids, since a username can
|
||||
// change on a whim and an id can't.
|
||||
|
||||
/** "Todding (@todding)" — the display name, then the handle when it differs. */
|
||||
function posterLabel(p) {
|
||||
const shown = p?.global_name || p?.username || p?.id || '';
|
||||
const handle = p?.username && p.username !== shown ? ` (@${p.username})` : '';
|
||||
return `${shown}${handle}`;
|
||||
}
|
||||
|
||||
/** What the picker starts with: the source's current list when it has one,
|
||||
* else the suggested creator(s). A set of ids. */
|
||||
function initialPosterPick(data) {
|
||||
const current = (data?.selected || []).map(String);
|
||||
if (current.length) return new Set(current);
|
||||
return new Set((data?.posters || []).filter((p) => p.suggested).map((p) => String(p.id)));
|
||||
}
|
||||
|
||||
/** The checklist's rows: who posted lately, then anyone already on the
|
||||
* source's list who hasn't — by the name they were picked under — so they
|
||||
* can still be unticked. */
|
||||
function withSelected(data) {
|
||||
const posters = [...(data?.posters || [])];
|
||||
const shown = new Set(posters.map((p) => String(p.id)));
|
||||
const labels = data?.selected_labels || {};
|
||||
for (const id of (data?.selected || []).map(String)) {
|
||||
if (shown.has(id)) continue;
|
||||
posters.push({ id, global_name: labels[id] || null, username: null,
|
||||
messages: 0, images: 0, reasons: ['on the list, not posting lately'] });
|
||||
}
|
||||
return posters;
|
||||
}
|
||||
|
||||
/** The picked posters as the API takes them: [{id, name}], in list order.
|
||||
* An id already on the source but no longer posting lately is kept. */
|
||||
function postersRequest(posters, picked) {
|
||||
const byId = new Map((posters || []).map((p) => [String(p.id), p]));
|
||||
return [...picked].map((id) => ({
|
||||
id,
|
||||
name: byId.has(id) ? (byId.get(id).global_name || byId.get(id).username || id) : null,
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -76,7 +76,7 @@ function updateConnectionDot(connected) {
|
||||
async function checkForUpdate() {
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({ type: 'CHECK_UPDATE' });
|
||||
if (r && r.updateAvailable && r.xpiUrl) showUpdateBanner(r);
|
||||
if (r && r.updateAvailable && r.installPageUrl) showUpdateBanner(r);
|
||||
} catch { /* non-fatal */ }
|
||||
}
|
||||
|
||||
@@ -86,10 +86,14 @@ function showUpdateBanner(r) {
|
||||
// exactly as it did before the field existed.
|
||||
const channel = r.channel ? ` (${r.channel})` : '';
|
||||
document.getElementById('update-text').textContent =
|
||||
`Update available${channel} — v${r.latestVersion} (installed v${r.currentVersion})`;
|
||||
// Opening the signed XPI triggers Firefox's native install prompt.
|
||||
`Update available${channel} — v${r.latestVersion} (installed v${r.currentVersion}). ` +
|
||||
'Opens FabledCurator — click “Install Firefox extension” there.';
|
||||
// Opens FC's install card rather than the XPI: Firefox only installs an
|
||||
// add-on from a user click on a web page, never from a tab an extension
|
||||
// opened on the .xpi itself.
|
||||
document.getElementById('update-btn').addEventListener('click', () => {
|
||||
browser.tabs.create({ url: r.xpiUrl });
|
||||
browser.tabs.create({ url: r.installPageUrl });
|
||||
window.close();
|
||||
});
|
||||
document.getElementById('update-banner').classList.remove('hidden');
|
||||
}
|
||||
|
||||
+149
-14
@@ -1,11 +1,12 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { loadLib } from './helpers/loadLib.js'
|
||||
|
||||
const { chipState, chipLabel, discordPanelDefaults, discordAddRequest } = loadLib('chip.js', [
|
||||
const { chipState, chipLabel, panelDefaults, addRequest, renameOffer } = loadLib('chip.js', [
|
||||
'chipState',
|
||||
'chipLabel',
|
||||
'discordPanelDefaults',
|
||||
'discordAddRequest'
|
||||
'panelDefaults',
|
||||
'addRequest',
|
||||
'renameOffer'
|
||||
])
|
||||
|
||||
const discord = (extra = {}) => ({
|
||||
@@ -63,46 +64,180 @@ describe('chip state and label', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('Discord Add panel', () => {
|
||||
describe('Add panel on Discord', () => {
|
||||
it('opens on the channel with the suggested artist preselected', () => {
|
||||
const d = discordPanelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } }))
|
||||
const d = panelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } }))
|
||||
expect(d.scope).toBe('channel')
|
||||
expect(d.artist).toEqual({ id: 7, name: 'Tamada' })
|
||||
expect(d.artistName).toBe('Tamada')
|
||||
})
|
||||
|
||||
it('proposes a new artist named after the server when nothing is suggested', () => {
|
||||
const d = discordPanelDefaults(discord({ state: 'new' }))
|
||||
const d = panelDefaults(discord({ state: 'new' }))
|
||||
expect(d.artist).toBe(null)
|
||||
expect(d.artistName).toBe('Studio')
|
||||
})
|
||||
|
||||
it('sends a picked artist by id', () => {
|
||||
const choice = discordPanelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } }))
|
||||
expect(discordAddRequest(choice)).toEqual({
|
||||
const choice = panelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } }))
|
||||
expect(addRequest(choice)).toEqual({
|
||||
url: 'https://discord.com/channels/111/222',
|
||||
artistId: 7
|
||||
})
|
||||
})
|
||||
|
||||
it('sends a typed name once the picked artist has been edited away', () => {
|
||||
const choice = discordPanelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } }))
|
||||
const choice = panelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } }))
|
||||
choice.artistName = 'Tamada Alt'
|
||||
expect(discordAddRequest(choice)).toEqual({
|
||||
expect(addRequest(choice)).toEqual({
|
||||
url: 'https://discord.com/channels/111/222',
|
||||
artistName: 'Tamada Alt'
|
||||
})
|
||||
})
|
||||
|
||||
it('adds the whole server when the operator picks it', () => {
|
||||
const choice = discordPanelDefaults(discord({ state: 'new' }))
|
||||
const choice = panelDefaults(discord({ state: 'new' }))
|
||||
choice.scope = 'server'
|
||||
expect(discordAddRequest(choice).url).toBe('https://discord.com/channels/111')
|
||||
expect(addRequest(choice).url).toBe('https://discord.com/channels/111')
|
||||
})
|
||||
|
||||
it('has nothing to send without an artist', () => {
|
||||
const choice = discordPanelDefaults(discord({ state: 'new' }))
|
||||
const choice = panelDefaults(discord({ state: 'new' }))
|
||||
choice.artistName = ' '
|
||||
expect(discordAddRequest(choice)).toBe(null)
|
||||
expect(addRequest(choice)).toBe(null)
|
||||
})
|
||||
})
|
||||
|
||||
const { squashName, exactArtistMatch, inlineCompletion } = loadLib('chip.js', [
|
||||
'squashName',
|
||||
'exactArtistMatch',
|
||||
'inlineCompletion'
|
||||
])
|
||||
|
||||
describe('artist matching for the Add panel', () => {
|
||||
const results = [
|
||||
{ id: 1, name: 'TamadaHeijun' },
|
||||
{ id: 2, name: 'Tamago' },
|
||||
{ id: 3, name: 'Sabu Art' }
|
||||
]
|
||||
|
||||
it('treats spacing, case and punctuation as the same name', () => {
|
||||
expect(squashName('Tamada Heijun')).toBe('tamadaheijun')
|
||||
expect(squashName('sabu_art!')).toBe('sabuart')
|
||||
expect(squashName('玉田 平順')).toBe('玉田平順')
|
||||
})
|
||||
|
||||
it('finds the result that is the query, spacing aside', () => {
|
||||
expect(exactArtistMatch('tamada heijun', results)).toEqual({ id: 1, name: 'TamadaHeijun' })
|
||||
expect(exactArtistMatch('Tama', results)).toBe(null)
|
||||
expect(exactArtistMatch(' ', results)).toBe(null)
|
||||
})
|
||||
|
||||
it('autofills the first name that extends what was typed', () => {
|
||||
expect(inlineCompletion('tamad', results)).toEqual({ id: 1, name: 'TamadaHeijun' })
|
||||
expect(inlineCompletion('Sab', results).name).toBe('Sabu Art')
|
||||
// Nothing to add once the name is complete, or when nothing extends it.
|
||||
expect(inlineCompletion('Sabu Art', results)).toBe(null)
|
||||
expect(inlineCompletion('heijun', results)).toBe(null)
|
||||
expect(inlineCompletion('', results)).toBe(null)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Add panel on Patreon and SubscribeStar', () => {
|
||||
const PAGE = 'https://www.patreon.com/cw/tamadaheijun'
|
||||
const patreon = (extra = {}) => ({ platform: 'patreon', slug: 'tamadaheijun', ...extra })
|
||||
|
||||
it('opens on the Patreon display name as a new artist', () => {
|
||||
const c = panelDefaults(patreon({ state: 'new', display_name: 'Tamada Heijun' }), PAGE)
|
||||
expect(c.scope).toBe('page')
|
||||
expect(c.artistName).toBe('Tamada Heijun')
|
||||
expect(addRequest(c)).toEqual({ url: PAGE, artistName: 'Tamada Heijun' })
|
||||
})
|
||||
|
||||
it('leaves an untouched URL handle to the server to resolve', () => {
|
||||
const c = panelDefaults(patreon({ state: 'new' }), PAGE)
|
||||
expect(c.artistName).toBe('tamadaheijun')
|
||||
expect(addRequest(c)).toEqual({ url: PAGE })
|
||||
c.artistName = 'Someone Else'
|
||||
expect(addRequest(c)).toEqual({ url: PAGE, artistName: 'Someone Else' })
|
||||
})
|
||||
|
||||
it('offers the Patreon name when joining an artist known by another name', () => {
|
||||
const c = panelDefaults(patreon({ state: 'new', display_name: 'Tamada Heijun' }), PAGE)
|
||||
c.artist = { id: 4, name: 'tamada' }
|
||||
c.artistName = 'tamada'
|
||||
expect(renameOffer(c)).toEqual({ from: 'tamada', to: 'Tamada Heijun' })
|
||||
expect(addRequest(c)).toEqual({ url: PAGE, artistId: 4, usePlatformName: true })
|
||||
c.adoptPlatformName = false
|
||||
expect(addRequest(c)).toEqual({ url: PAGE, artistId: 4 })
|
||||
})
|
||||
|
||||
it('offers no rename when the names agree or the name was not read', () => {
|
||||
const same = panelDefaults(patreon({ state: 'new', display_name: 'Tamada Heijun' }), PAGE)
|
||||
same.artist = { id: 4, name: 'Tamada Heijun' }
|
||||
expect(renameOffer(same)).toBe(null)
|
||||
const unread = panelDefaults(patreon({ state: 'new' }), PAGE)
|
||||
unread.artist = { id: 4, name: 'tamada' }
|
||||
expect(renameOffer(unread)).toBe(null)
|
||||
})
|
||||
|
||||
it('never renames from SubscribeStar or Discord — Patreon is the canon', () => {
|
||||
const ss = panelDefaults(
|
||||
{ platform: 'subscribestar', slug: 'tamada', state: 'new', display_name: 'SS Tamada' },
|
||||
'https://subscribestar.adult/tamada'
|
||||
)
|
||||
ss.artist = { id: 4, name: 'Tamada Heijun' }
|
||||
ss.artistName = 'Tamada Heijun'
|
||||
expect(renameOffer(ss)).toBe(null)
|
||||
expect(addRequest(ss)).toEqual({ url: 'https://subscribestar.adult/tamada', artistId: 4 })
|
||||
})
|
||||
|
||||
it('preselects the artist the URL already names', () => {
|
||||
const c = panelDefaults(patreon({ state: 'artist_match', artist: { id: 9, name: 'Tamada' } }), PAGE)
|
||||
expect(c.artist).toEqual({ id: 9, name: 'Tamada' })
|
||||
expect(c.artistName).toBe('Tamada')
|
||||
})
|
||||
})
|
||||
|
||||
const { posterLabel, initialPosterPick, postersRequest, withSelected } = loadLib('chip.js', [
|
||||
'posterLabel', 'initialPosterPick', 'postersRequest', 'withSelected',
|
||||
])
|
||||
|
||||
describe('the poster checklist rows', () => {
|
||||
it('adds someone on the list who has not posted lately, by their saved name', () => {
|
||||
const rows = withSelected({
|
||||
posters: [{ id: '7', username: 'todding', global_name: 'Todding' }],
|
||||
selected: ['7', '42'],
|
||||
selected_labels: { 42: 'Old Name' },
|
||||
})
|
||||
expect(rows.map((r) => r.id)).toEqual(['7', '42'])
|
||||
expect(posterLabel(rows[1])).toBe('Old Name')
|
||||
expect(rows[1].reasons).toEqual(['on the list, not posting lately'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('the Discord poster picker (#4491)', () => {
|
||||
const posters = [
|
||||
{ id: '7', username: 'todding', global_name: 'Todding', suggested: true },
|
||||
{ id: '8', username: 'jakeboii', global_name: 'Jake Boii', suggested: false },
|
||||
{ id: '9', username: 'plain', global_name: null, suggested: false },
|
||||
]
|
||||
|
||||
it('labels a poster by display name, then handle', () => {
|
||||
expect(posterLabel(posters[0])).toBe('Todding (@todding)')
|
||||
expect(posterLabel(posters[2])).toBe('plain')
|
||||
})
|
||||
|
||||
it('starts from the source list when it has one, else the suggestion', () => {
|
||||
expect([...initialPosterPick({ posters, selected: [] })]).toEqual(['7'])
|
||||
expect([...initialPosterPick({ posters, selected: ['8'] })]).toEqual(['8'])
|
||||
expect([...initialPosterPick({ posters: [] })]).toEqual([])
|
||||
})
|
||||
|
||||
it('sends ids with the names they were picked under, keeping unknown ids', () => {
|
||||
expect(postersRequest(posters, new Set(['7', '42']))).toEqual([
|
||||
{ id: '7', name: 'Todding' },
|
||||
{ id: '42', name: null },
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -101,7 +101,7 @@ import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||
import { useCleanupStore } from '../../stores/cleanup.js'
|
||||
|
||||
const store = useCleanupStore()
|
||||
const threshold = ref(0.95)
|
||||
const threshold = ref(0.995)
|
||||
const tolerance = ref(30)
|
||||
const audit = ref(null)
|
||||
const busy = ref(false)
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
<!-- System-tag auto-applies (chrome hides / process WIP tags) that ALSO looked
|
||||
like real content — surfaced PROACTIVELY atop the gallery whenever there's
|
||||
something to review (NOT gated on the Show-hidden toggle, so misfires can't
|
||||
go unnoticed), most-concerning first, with keep / remove (#141, #1464).
|
||||
go unnoticed), most-concerning first (#141, #1464). Each card asks one
|
||||
question — is this image a <tag>? — and its buttons answer it (#4424).
|
||||
Renders nothing when there's nothing to review. -->
|
||||
<section v-if="items.length" class="fc-review" aria-label="Auto-tagged images to review">
|
||||
<div class="fc-review__head">
|
||||
<v-icon size="18" color="warning">mdi-alert-outline</v-icon>
|
||||
<span class="fc-review__title">
|
||||
{{ items.length }} auto-tagged {{ items.length === 1 ? 'image' : 'images' }}
|
||||
may be real content — review
|
||||
{{ items.length }} {{ items.length === 1 ? 'auto-tag' : 'auto-tags' }} to check
|
||||
</span>
|
||||
</div>
|
||||
<div class="fc-review__cards">
|
||||
@@ -21,22 +21,23 @@
|
||||
class="fc-review-card__thumb" loading="lazy"
|
||||
>
|
||||
<div class="fc-review-card__body">
|
||||
<div class="fc-review-card__question">{{ question(it) }}</div>
|
||||
<div
|
||||
class="fc-review-card__conflict"
|
||||
:title="`Scored ${Math.round(it.conflict_score * 100)}% on “${it.conflict_name || 'a content tag'}”`"
|
||||
class="fc-review-card__reason"
|
||||
:title="reasonTitle(it)"
|
||||
>
|
||||
also looks like <strong>{{ it.conflict_name || 'content' }}</strong>
|
||||
also {{ Math.round(it.conflict_score * 100) }}%
|
||||
<strong>{{ it.conflict_name || 'content' }}</strong>
|
||||
</div>
|
||||
<div class="fc-review-card__tag">{{ tagLine(it) }}</div>
|
||||
<div class="fc-review-card__acts">
|
||||
<button
|
||||
type="button" class="fc-review-btn fc-review-btn--keep"
|
||||
:disabled="busy.includes(keyOf(it))" @click="resolve(it, 'keep')"
|
||||
>{{ keepLabel(it) }}</button>
|
||||
>Is {{ withArticle(it) }}</button>
|
||||
<button
|
||||
type="button" class="fc-review-btn fc-review-btn--unhide"
|
||||
:disabled="busy.includes(keyOf(it))" @click="resolve(it, 'unhide')"
|
||||
>{{ removeLabel(it) }}</button>
|
||||
>Is not {{ withArticle(it) }}</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -55,11 +56,18 @@ const items = ref([])
|
||||
const busy = ref([])
|
||||
|
||||
function keyOf(it) { return `${it.image_id}:${it.tag_id}` }
|
||||
// Chrome flags hide the image (keep-hidden / un-hide); process flags leave it
|
||||
// visible and just tagged (keep-tag / remove-tag). Same endpoints, different words.
|
||||
function tagLine(it) { return (it.mode === 'process' ? 'auto-tagged ' : 'hidden as ') + it.tag_name }
|
||||
function keepLabel(it) { return it.mode === 'process' ? 'Keep tag' : 'Keep hidden' }
|
||||
function removeLabel(it) { return it.mode === 'process' ? 'Remove tag' : 'Un-hide' }
|
||||
// The card asks whether the image IS the auto-applied system tag, and the buttons
|
||||
// answer that (operator, #4424: "is a <tag>" / "is not a <tag>"). "Is" keeps the
|
||||
// tag ('keep'); "Is not" removes it, un-hiding a chrome image ('unhide'). The
|
||||
// content tag it also scored on is the reason it was flagged, not the question.
|
||||
function noun(it) { return it.tag_name === 'wip' ? 'WIP' : it.tag_name }
|
||||
function withArticle(it) { return (/^[aeiou]/i.test(noun(it)) ? 'an ' : 'a ') + noun(it) }
|
||||
function question(it) { return `Is this ${withArticle(it)}?` }
|
||||
function reasonTitle(it) {
|
||||
const hidden = it.mode === 'process' ? '' : ' It is hidden from the gallery until you answer.'
|
||||
return `Auto-tagged “${it.tag_name}”, but it also scored ${Math.round(it.conflict_score * 100)}% `
|
||||
+ `on “${it.conflict_name || 'a content tag'}”, so it may be finished art.${hidden}`
|
||||
}
|
||||
|
||||
async function load() {
|
||||
// Fetched unconditionally on mount — the strip prompts for pending misfires
|
||||
@@ -77,14 +85,11 @@ async function resolve(it, action) {
|
||||
await api.post(`/api/gallery/hidden-review/${it.image_id}/${it.tag_id}/${action}`)
|
||||
items.value = items.value.filter((x) => keyOf(x) !== k)
|
||||
if (action === 'unhide') {
|
||||
const verb = it.mode === 'process' ? 'Removed' : 'Un-hidden'
|
||||
toast({ text: `${verb} — “${it.tag_name}” removed; it'll train the head`, type: 'success' })
|
||||
const shown = it.mode === 'process' ? '' : ', back in the gallery'
|
||||
toast({ text: `Not ${withArticle(it)} — “${it.tag_name}” removed${shown}; the tagger learns from it`, type: 'success' })
|
||||
}
|
||||
} catch (e) {
|
||||
toast({
|
||||
text: `Could not ${action === 'keep' ? 'keep hidden' : 'un-hide'}: ${e.message}`,
|
||||
type: 'error',
|
||||
})
|
||||
toast({ text: `Could not save your answer: ${e.message}`, type: 'error' })
|
||||
} finally {
|
||||
busy.value = busy.value.filter((x) => x !== k)
|
||||
}
|
||||
@@ -112,7 +117,7 @@ onMounted(load)
|
||||
display: flex; gap: 10px; overflow-x: auto; padding-bottom: 4px;
|
||||
}
|
||||
.fc-review-card {
|
||||
flex: 0 0 auto; width: 150px;
|
||||
flex: 0 0 auto; width: 170px;
|
||||
display: flex; flex-direction: column;
|
||||
border: 1px solid rgb(var(--v-theme-surface-light));
|
||||
border-radius: 6px; overflow: hidden;
|
||||
@@ -123,16 +128,16 @@ onMounted(load)
|
||||
background: rgb(var(--v-theme-surface-light));
|
||||
}
|
||||
.fc-review-card__body { padding: 6px 8px; }
|
||||
.fc-review-card__conflict {
|
||||
font-size: 11px; color: rgb(var(--v-theme-on-surface));
|
||||
.fc-review-card__question {
|
||||
font-size: 12px; font-weight: 600; color: rgb(var(--v-theme-on-surface));
|
||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||
}
|
||||
.fc-review-card__conflict strong { color: rgb(var(--v-theme-warning)); }
|
||||
.fc-review-card__tag {
|
||||
.fc-review-card__reason {
|
||||
font-size: 10px; color: rgb(var(--v-theme-on-surface-variant));
|
||||
margin: 1px 0 6px;
|
||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||
}
|
||||
.fc-review-card__reason strong { color: rgb(var(--v-theme-warning)); font-weight: 600; }
|
||||
.fc-review-card__acts { display: flex; gap: 4px; }
|
||||
.fc-review-btn {
|
||||
flex: 1; font-size: 11px; padding: 3px 4px; border-radius: 4px;
|
||||
|
||||
@@ -58,7 +58,7 @@ import { computed, ref, watch } from 'vue'
|
||||
const props = defineProps({
|
||||
modelValue: { type: Boolean, required: true },
|
||||
action: { type: String, required: true }, // 'restore' | 'delete'
|
||||
kind: { type: String, required: true }, // 'db' | 'images' | 'artist' | 'tag' | 'images-selection' | 'audit' | 'min-dim'
|
||||
kind: { type: String, required: true }, // 'db' | 'offsite' | 'artist' | 'tag' | 'images-selection' | 'audit' | 'min-dim'
|
||||
runId: { type: [Number, String], default: '' }, // numeric id or sha8 string
|
||||
description: { type: String, default: '' },
|
||||
tier: { type: String, default: 'C' }, // 'B' | 'C'
|
||||
@@ -89,6 +89,7 @@ const kindLabel = computed(() => ({
|
||||
artist: 'artist',
|
||||
tag: 'tag',
|
||||
'images-selection': 'image selection',
|
||||
offsite: 'from the off-site bucket',
|
||||
}[props.kind] || props.kind))
|
||||
const alertType = computed(
|
||||
() => props.action === 'restore' ? 'warning' : 'error',
|
||||
|
||||
@@ -2,11 +2,15 @@
|
||||
<div
|
||||
ref="canvasEl"
|
||||
class="fc-canvas"
|
||||
:class="{ 'fc-canvas--zoomed': panZoom.state.scale > 1 }"
|
||||
:class="{
|
||||
'fc-canvas--zoomed': panZoom.state.scale > 1,
|
||||
'fc-canvas--moving': !panZoom.state.smooth,
|
||||
}"
|
||||
@wheel="panZoom.handlers.onWheel"
|
||||
@pointerdown="panZoom.handlers.onPointerDown"
|
||||
@pointermove="panZoom.handlers.onPointerMove"
|
||||
@pointerup="panZoom.handlers.onPointerUp"
|
||||
@pointercancel="panZoom.handlers.onPointerCancel"
|
||||
@click="onCanvasClick"
|
||||
>
|
||||
<img
|
||||
@@ -18,6 +22,34 @@
|
||||
draggable="false"
|
||||
@load="onImgLoad"
|
||||
>
|
||||
<!-- Zoom controls. They sit on the canvas but are not part of it: a press,
|
||||
click or scroll on them never pans, toggles or closes. -->
|
||||
<div
|
||||
class="fc-canvas__zoombar"
|
||||
@pointerdown.stop @click.stop @wheel.stop
|
||||
>
|
||||
<button
|
||||
type="button" class="fc-canvas__zoombtn" aria-label="Zoom out"
|
||||
:disabled="panZoom.state.scale <= 1" @click="panZoom.zoomBy(-1)"
|
||||
><v-icon size="18">mdi-minus</v-icon></button>
|
||||
<input
|
||||
type="range" class="fc-canvas__zoomslider" aria-label="Zoom"
|
||||
min="0" max="1" step="0.001" :value="sliderValue"
|
||||
@input="onSlider"
|
||||
>
|
||||
<button
|
||||
type="button" class="fc-canvas__zoombtn" aria-label="Zoom in"
|
||||
:disabled="panZoom.state.scale >= maxScale" @click="panZoom.zoomBy(1)"
|
||||
><v-icon size="18">mdi-plus</v-icon></button>
|
||||
<span class="fc-canvas__zoomlabel" :title="'Shown at ' + zoomLabel + ' of its full size'">
|
||||
{{ zoomLabel }}
|
||||
</span>
|
||||
<button
|
||||
type="button" class="fc-canvas__zoombtn" aria-label="Fit to the window"
|
||||
title="Fit to the window"
|
||||
:disabled="panZoom.state.scale <= 1" @click="panZoom.reset()"
|
||||
><v-icon size="18">mdi-fit-to-screen-outline</v-icon></button>
|
||||
</div>
|
||||
<!-- #1206 grounding overlay: tracks the <img>'s live rendered rect (so it's
|
||||
correct under object-fit letterboxing + pan/zoom) and draws the crop
|
||||
region a hovered suggestion came from. Null grounding → a subtle
|
||||
@@ -60,10 +92,37 @@ const props = defineProps({
|
||||
hover: { type: Object, default: null },
|
||||
})
|
||||
const emit = defineEmits(['close-request'])
|
||||
const panZoom = usePanZoom()
|
||||
|
||||
const canvasEl = ref(null)
|
||||
const imgEl = ref(null)
|
||||
// Where the picture sits before any zoom: the canvas size, the picture's
|
||||
// layout box (the zoom is a transform on top of it) and its natural width.
|
||||
// Read again when the picture loads and when the window resizes.
|
||||
const fitted = ref(null)
|
||||
function refit() {
|
||||
const img = imgEl.value
|
||||
const canvas = canvasEl.value
|
||||
fitted.value = img && canvas ? {
|
||||
cw: canvas.clientWidth, ch: canvas.clientHeight,
|
||||
iw: img.offsetWidth, ih: img.offsetHeight, nw: img.naturalWidth,
|
||||
} : null
|
||||
panZoom.clamp()
|
||||
}
|
||||
const panZoom = usePanZoom(() => fitted.value)
|
||||
|
||||
// The slider runs on a log scale, so each stretch of it is the same zoom step
|
||||
// whether the picture is at 1.2x or 12x.
|
||||
const maxScale = computed(() => panZoom.maxScale())
|
||||
const sliderValue = computed(() =>
|
||||
Math.log(panZoom.state.scale) / Math.log(maxScale.value))
|
||||
function onSlider(ev) {
|
||||
panZoom.setScale(maxScale.value ** Number(ev.target.value))
|
||||
}
|
||||
// How big it is against its full-size pixels.
|
||||
const zoomLabel = computed(() => {
|
||||
const g = fitted.value
|
||||
const pct = g?.iw && g?.nw ? (g.iw * panZoom.state.scale) / g.nw : panZoom.state.scale
|
||||
return `${Math.round(pct * 100)}%`
|
||||
})
|
||||
const overlayBox = ref(null) // the <img>'s rect relative to the canvas
|
||||
|
||||
// The <img> renders aspect-fit + transformed, so its getBoundingClientRect is
|
||||
@@ -107,22 +166,21 @@ watch(
|
||||
() => [panZoom.state.scale, panZoom.state.x, panZoom.state.y],
|
||||
() => { if (props.hover) measure() },
|
||||
)
|
||||
function onResize () { if (props.hover) measure() }
|
||||
function onImgLoad () { if (props.hover) measure() }
|
||||
function onResize () { refit(); if (props.hover) measure() }
|
||||
function onImgLoad () { refit(); if (props.hover) measure() }
|
||||
onMounted(() => window.addEventListener('resize', onResize))
|
||||
onUnmounted(() => window.removeEventListener('resize', onResize))
|
||||
|
||||
// Reset zoom when the src changes (prev/next nav).
|
||||
watch(() => props.src, () => panZoom.reset())
|
||||
|
||||
// Click on the image → toggle zoom (IR/expected behavior). Click on
|
||||
// the haze area around the image → request close. We use the image's
|
||||
// own bounding rect rather than @click.self because the <img> sets
|
||||
// pointer-events: none for the pan-drag pathway, so the click always
|
||||
// targets the canvas div regardless of where the cursor was.
|
||||
// Click on the image → toggle zoom. Click on the haze around it → request
|
||||
// close. The <img> has pointer-events: none so a drag always lands on the
|
||||
// canvas, which is why "on the image" is worked out from its rect. The click
|
||||
// that ends a drag does neither.
|
||||
function onCanvasClick(ev) {
|
||||
if (panZoom.endedDrag()) return
|
||||
if (panZoom.state.scale > 1) {
|
||||
// When zoomed, any click resets zoom — preserve old behavior.
|
||||
panZoom.handlers.onClick(ev)
|
||||
return
|
||||
}
|
||||
@@ -160,6 +218,49 @@ function onCanvasClick(ev) {
|
||||
transition: transform 100ms ease-out;
|
||||
pointer-events: none;
|
||||
}
|
||||
.fc-canvas--moving img { transition: none; }
|
||||
|
||||
/* Zoom controls, bottom centre of the canvas: quiet until the canvas is
|
||||
hovered or the picture is zoomed. */
|
||||
.fc-canvas__zoombar {
|
||||
position: absolute;
|
||||
bottom: 16px; left: 50%;
|
||||
transform: translateX(-50%);
|
||||
z-index: 3;
|
||||
display: flex; align-items: center; gap: 4px;
|
||||
padding: 4px 8px;
|
||||
border-radius: 999px;
|
||||
background: rgba(var(--v-theme-background), 0.7);
|
||||
border: 1px solid rgb(var(--v-theme-surface-light));
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
cursor: default;
|
||||
opacity: 0;
|
||||
transition: opacity 150ms ease;
|
||||
}
|
||||
.fc-canvas:hover .fc-canvas__zoombar,
|
||||
.fc-canvas--zoomed .fc-canvas__zoombar,
|
||||
.fc-canvas__zoombar:focus-within { opacity: 1; }
|
||||
.fc-canvas__zoombtn {
|
||||
width: 28px; height: 28px;
|
||||
display: grid; place-items: center;
|
||||
border-radius: 50%;
|
||||
color: inherit;
|
||||
cursor: pointer;
|
||||
}
|
||||
.fc-canvas__zoombtn:hover:not(:disabled) { background: rgb(var(--v-theme-surface)); }
|
||||
.fc-canvas__zoombtn:disabled { opacity: 0.35; cursor: default; }
|
||||
.fc-canvas__zoomslider {
|
||||
width: 140px;
|
||||
accent-color: rgb(var(--v-theme-accent));
|
||||
cursor: pointer;
|
||||
}
|
||||
.fc-canvas__zoomlabel {
|
||||
min-width: 44px;
|
||||
text-align: center;
|
||||
font-size: 12px;
|
||||
font-variant-numeric: tabular-nums;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
|
||||
/* Grounding overlay — never intercepts pan/zoom/click. */
|
||||
.fc-canvas__overlay {
|
||||
|
||||
@@ -18,6 +18,9 @@
|
||||
<h3>Keyboard shortcuts</h3>
|
||||
<dl>
|
||||
<div><dt>← / →</dt><dd>Previous / next image</dd></div>
|
||||
<div><dt>Click</dt><dd>Zoom in (a long strip opens at full width) / back to fit</dd></div>
|
||||
<div><dt>Scroll · drag</dt><dd>Move around a zoomed image</dd></div>
|
||||
<div><dt>Ctrl + scroll</dt><dd>Zoom at the pointer</dd></div>
|
||||
<div><dt>T or /</dt><dd>Jump to the tag input</dd></div>
|
||||
<div><dt>↑ / ↓</dt><dd>Move through tag suggestions</dd></div>
|
||||
<div><dt>Enter / Tab</dt><dd>Accept the highlighted tag</dd></div>
|
||||
|
||||
@@ -26,6 +26,7 @@
|
||||
:to="{ name: 'artist', params: { slug: post.artist.slug } }"
|
||||
class="fc-post-card__artist"
|
||||
>{{ post.artist.name }}</RouterLink>
|
||||
<span v-if="post.channel" class="fc-post-card__meta">#{{ post.channel }}</span>
|
||||
<span class="fc-post-card__date" :title="absoluteDate">{{ relativeDate }}</span>
|
||||
<span v-if="totalImages" class="fc-post-card__meta">
|
||||
· {{ totalImages }} image{{ totalImages === 1 ? '' : 's' }}
|
||||
|
||||
@@ -1,23 +1,21 @@
|
||||
<template>
|
||||
<!-- #713: re-extract PostAttachments that are really archives but were filed
|
||||
opaquely before the magic-byte gate, and attach their images to the post. -->
|
||||
<MaintenanceTile
|
||||
icon="mdi-folder-zip-outline"
|
||||
title="Re-extract archives"
|
||||
blurb="Extract images from zip attachments filed opaquely."
|
||||
blurb="Import the images inside archive attachments that were never extracted."
|
||||
:open="busy"
|
||||
>
|
||||
<p class="text-body-2 mb-3">
|
||||
Some posts attached an archive (zip) whose images weren't extracted
|
||||
because the downloaded file had no usable extension. This scans existing
|
||||
attachments, extracts any that are really archives, and attaches their
|
||||
images to the post. Idempotent — safe to run more than once.
|
||||
Goes through every stored zip, rar and 7z, including ones imported from
|
||||
a folder without a post, and imports the images inside. Archives already
|
||||
extracted are passed over, and game or app builds stay whole. Safe to
|
||||
run again.
|
||||
</p>
|
||||
<v-btn color="primary" rounded="pill" :loading="busy" @click="run">
|
||||
<v-icon start>mdi-folder-zip-outline</v-icon> Re-extract archives now
|
||||
</v-btn>
|
||||
<span v-if="queued" class="ml-3 text-caption text-success">Queued ✓</span>
|
||||
<QueueStatusBar queue="maintenance" queue-label="Maintenance" />
|
||||
<QueueStatusBar queue="maintenance_long" queue-label="Long maintenance" />
|
||||
</MaintenanceTile>
|
||||
</template>
|
||||
|
||||
|
||||
@@ -2,13 +2,19 @@
|
||||
<MaintenanceTile
|
||||
icon="mdi-database-export"
|
||||
title="Backups"
|
||||
blurb="Database + image backups, schedules and restore."
|
||||
blurb="Database and library backups, schedules and restore."
|
||||
>
|
||||
<p class="fc-muted text-body-2 mb-4">
|
||||
pg_dump for the database (fast, nightly-schedulable);
|
||||
tar+zstd for the images (slow, manual only). Files live in
|
||||
<code>/images/_backups/</code>. Tag a backup to protect it
|
||||
from autoprune.
|
||||
<p v-if="offsite" class="fc-muted text-body-2 mb-4">
|
||||
Database dumps stream to the <strong>{{ offsite.bucket }}</strong> bucket
|
||||
at {{ offsite.endpoint }}, with no copy kept on this server; the bucket's
|
||||
own retention thins old dumps. Images and attachments go to the same
|
||||
bucket's <code>media/</code> folder, below.
|
||||
</p>
|
||||
<p v-else class="fc-muted text-body-2 mb-4">
|
||||
Database dumps are written to <code>/images/_backups/</code> on this
|
||||
server; tag one to keep it past Keep last. Images and attachments are
|
||||
backed up only to an off-site destination: set one up above to back up
|
||||
the whole library, database included, off this server.
|
||||
</p>
|
||||
|
||||
<v-alert
|
||||
@@ -18,6 +24,21 @@
|
||||
{{ store.lastError }}
|
||||
</v-alert>
|
||||
|
||||
<!-- Whether the backups are keeping up: the same verdict as the Backups
|
||||
row on the System tab and the dot beside the brand. -->
|
||||
<v-alert
|
||||
v-if="backupPart && backupPart.state !== 'ok'"
|
||||
type="warning" variant="tonal" density="compact" class="mb-3"
|
||||
>{{ backupPart.detail }}.</v-alert>
|
||||
<div
|
||||
v-else-if="backupPart" class="d-flex align-center text-body-2 mb-3"
|
||||
style="font-variant-numeric: tabular-nums;"
|
||||
>
|
||||
<v-icon size="small" color="success" class="mr-2">mdi-check-circle</v-icon>
|
||||
{{ backupPart.detail }}<template v-if="newestDump?.size_bytes">,
|
||||
{{ humanBytes(newestDump.size_bytes) }}</template>.
|
||||
</div>
|
||||
|
||||
<!-- Database section -->
|
||||
<h3 class="fc-section-title">Database</h3>
|
||||
<div class="fc-settings-row mb-3">
|
||||
@@ -40,6 +61,10 @@
|
||||
@update:model-value="onSettingChange('backup_db_keep_last_n', Number($event))"
|
||||
/>
|
||||
</div>
|
||||
<p v-if="offsite" class="fc-muted text-caption mb-3">
|
||||
Nightly sends the database, then any new images and attachments. Keep
|
||||
last applies to this list and to any dumps still on this server.
|
||||
</p>
|
||||
<v-btn
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-database-arrow-up" class="mb-3"
|
||||
@@ -52,81 +77,261 @@
|
||||
@restore="onRestore"
|
||||
@delete="onDelete"
|
||||
@tag="onTag"
|
||||
@test="(r) => testSection?.start({ runId: r.id })"
|
||||
/>
|
||||
|
||||
<v-divider class="my-4" />
|
||||
|
||||
<!-- Images section -->
|
||||
<h3 class="fc-section-title">Images</h3>
|
||||
<div class="fc-settings-row mb-3">
|
||||
<v-text-field
|
||||
:model-value="settings.backup_images_keep_last_n"
|
||||
label="Keep last" type="number" min="1" max="100"
|
||||
density="compact" hide-details style="max-width: 110px;"
|
||||
@update:model-value="onSettingChange('backup_images_keep_last_n', Number($event))"
|
||||
<BackupTestSection ref="testSection" :bucket="offsite?.bucket || ''" @started="pollOnce" />
|
||||
|
||||
<v-divider class="my-4" />
|
||||
|
||||
<!-- Off-site: images and attachments as one file per fingerprint -->
|
||||
<template v-if="offsite">
|
||||
<h3 class="fc-section-title">Images and attachments</h3>
|
||||
<p class="fc-muted text-body-2 mb-2">
|
||||
Each file goes to the bucket's <code>media/</code> folder once, named by
|
||||
its fingerprint, and the bucket checks it against the fingerprint taken
|
||||
at import. Only files not sent yet go in each run. With Nightly on, they
|
||||
follow the database dump every night.
|
||||
</p>
|
||||
<div v-if="coverage" class="mb-3">
|
||||
<div class="text-body-2 mb-1" style="font-variant-numeric: tabular-nums;">
|
||||
{{ coverage.files_sent.toLocaleString() }} of
|
||||
{{ coverage.files_total.toLocaleString() }} files in the bucket
|
||||
({{ humanBytes(coverage.bytes_sent) }} of {{ humanBytes(coverage.bytes_total) }})
|
||||
</div>
|
||||
<v-progress-linear
|
||||
:model-value="coveragePct" color="accent" height="6" rounded
|
||||
/>
|
||||
</div>
|
||||
<v-btn
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-cloud-upload-outline" class="mb-3"
|
||||
:loading="mediaTriggering" :disabled="mediaRunning"
|
||||
@click="onTrigger('media')"
|
||||
>Send new files now</v-btn>
|
||||
<span v-if="mediaRunning" class="text-caption fc-muted ml-2">
|
||||
A run is going: {{ runningSent.toLocaleString() }} files
|
||||
({{ humanBytes(runningMedia.size_bytes || 0) }}) sent so far.
|
||||
</span>
|
||||
<p class="fc-muted text-caption mb-2" style="font-variant-numeric: tabular-nums;">
|
||||
{{ lastCheckLine }}
|
||||
</p>
|
||||
<BackupRunsTable :runs="store.mediaRuns" />
|
||||
|
||||
<v-divider class="my-4" />
|
||||
|
||||
<!-- Off-site: restoring this install from the bucket -->
|
||||
<h3 class="fc-section-title">Restore from the bucket</h3>
|
||||
<p class="fc-muted text-body-2 mb-2">
|
||||
Replace this install's database with one of the bucket's dumps, then
|
||||
fetch every image and attachment back to where it belongs, each
|
||||
checked against its fingerprint as it arrives. The database being
|
||||
replaced is saved to the bucket first, so a restore can be undone.
|
||||
Account logins aren't in the backup: sign in to each again afterwards.
|
||||
</p>
|
||||
<p class="fc-muted text-body-2 mb-2">
|
||||
To repair a library that lost files, fetch only what's missing from
|
||||
disk; the database stays as it is.
|
||||
</p>
|
||||
<div class="fc-settings-row mb-3">
|
||||
<v-btn
|
||||
color="warning" variant="tonal" rounded="pill"
|
||||
prepend-icon="mdi-database-arrow-down"
|
||||
:disabled="anyGoing" :loading="dumpsLoading"
|
||||
@click="openDumps"
|
||||
>Choose a dump…</v-btn>
|
||||
<v-btn
|
||||
variant="tonal" rounded="pill"
|
||||
prepend-icon="mdi-cloud-download-outline"
|
||||
:disabled="anyGoing" :loading="fetchTriggering"
|
||||
@click="onFetchMissing"
|
||||
>Fetch missing files</v-btn>
|
||||
</div>
|
||||
<v-alert
|
||||
v-if="restoreDispatched || runningRestore"
|
||||
type="info" variant="tonal" density="compact" class="mb-3"
|
||||
>
|
||||
<template v-if="runningRestore?.manifest?.phase === 'files'">
|
||||
Fetching files: {{ restoreLine(runningRestore.manifest) }}.
|
||||
</template>
|
||||
<template v-else>
|
||||
Restoring the database. While it loads, this page may stop answering
|
||||
for a few minutes; it comes back on its own.
|
||||
</template>
|
||||
</v-alert>
|
||||
<BackupRunsTable :runs="store.restoreRuns" empty-text="No restores yet." />
|
||||
|
||||
<DumpPickerDialog
|
||||
v-model="dumpsOpen" action="restore"
|
||||
:bucket="offsite.bucket" :dumps="dumps" :error="dumpsError"
|
||||
@pick="onPickDump"
|
||||
/>
|
||||
</div>
|
||||
<v-btn
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-folder-zip" class="mb-3"
|
||||
:loading="imagesTriggering"
|
||||
@click="onTrigger('images')"
|
||||
>Run images backup now</v-btn>
|
||||
<span class="text-caption fc-muted ml-2">
|
||||
ⓘ Hours for a 200GB+ library.
|
||||
</span>
|
||||
</template>
|
||||
|
||||
<BackupRunsTable
|
||||
:runs="store.imagesRuns"
|
||||
@restore="onRestore"
|
||||
@delete="onDelete"
|
||||
@tag="onTag"
|
||||
/>
|
||||
<BackupConfirmModal
|
||||
v-model="confirmOpen"
|
||||
:action="confirmAction"
|
||||
:kind="confirmKind"
|
||||
:run-id="confirmRunId"
|
||||
:description="confirmDescription"
|
||||
:expected-token-override="confirmToken"
|
||||
@confirm="onConfirmSubmit"
|
||||
/>
|
||||
</MaintenanceTile>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||
import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
|
||||
|
||||
import { useBackupStore } from '../../stores/backup.js'
|
||||
import { useSystemHealthStore } from '../../stores/systemHealth.js'
|
||||
import BackupConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||
import { humanBytes } from '../../utils/bytes.js'
|
||||
import { formatDumpTime, formatRelative } from '../../utils/date.js'
|
||||
import BackupRunsTable from './BackupRunsTable.vue'
|
||||
import BackupTestSection from './BackupTestSection.vue'
|
||||
import DumpPickerDialog from './DumpPickerDialog.vue'
|
||||
|
||||
const store = useBackupStore()
|
||||
const healthStore = useSystemHealthStore()
|
||||
|
||||
// The server's verdict on the backups; absent when none are set up.
|
||||
const backupPart = computed(() => healthStore.parts.find((p) => p.key === 'backup'))
|
||||
const newestDump = computed(() => store.dbRuns.find((r) => r.status === 'ok'))
|
||||
|
||||
// The weekly check of the bucket against what was sent.
|
||||
const lastCheckLine = computed(() => {
|
||||
const r = store.reconcileRuns.find((x) => x.status !== 'running')
|
||||
if (!r) return 'The bucket is checked against what was sent once a week; no check has run yet.'
|
||||
const when = formatRelative(r.finished_at || r.started_at)
|
||||
if (r.status === 'error') return `The last bucket check (${when}) failed: ${r.error}`
|
||||
const n = (k) => (r.manifest?.[k] || 0).toLocaleString()
|
||||
const missing = r.manifest?.dropped
|
||||
? `${n('dropped')} missing, sent again by the next run`
|
||||
: 'none missing'
|
||||
return `Bucket checked ${when}: ${n('listed')} files there, ${missing}.`
|
||||
})
|
||||
|
||||
const settings = computed(() => store.settings || {
|
||||
backup_db_nightly_enabled: false,
|
||||
backup_db_nightly_hour_utc: 3,
|
||||
backup_db_keep_last_n: 14,
|
||||
backup_images_keep_last_n: 3,
|
||||
})
|
||||
|
||||
// The off-site destination, once it is fully set up: database dumps go there.
|
||||
const offsite = computed(() => (
|
||||
store.destination?.configured ? store.destination : null
|
||||
))
|
||||
|
||||
const coverage = computed(() => store.mediaCoverage)
|
||||
const coveragePct = computed(() => {
|
||||
const c = coverage.value
|
||||
return c && c.files_total ? (100 * c.files_sent) / c.files_total : 0
|
||||
})
|
||||
const runningMedia = computed(() => store.mediaRuns.find((r) => r.status === 'running'))
|
||||
const mediaRunning = computed(() => Boolean(runningMedia.value))
|
||||
const runningSent = computed(() => runningMedia.value?.manifest?.sent || 0)
|
||||
|
||||
const mediaTriggering = ref(false)
|
||||
const dbTriggering = ref(false)
|
||||
const imagesTriggering = ref(false)
|
||||
const confirmOpen = ref(false)
|
||||
const confirmAction = ref('restore')
|
||||
const confirmKind = ref('db')
|
||||
const confirmRunId = ref(0)
|
||||
const confirmDescription = ref('')
|
||||
const confirmToken = ref('')
|
||||
|
||||
// Restoring from the bucket.
|
||||
const runningRestore = computed(() => store.restoreRuns.find((r) => r.status === 'restoring'))
|
||||
const anyGoing = computed(() => Boolean(
|
||||
runningRestore.value || mediaRunning.value
|
||||
|| store.dbRuns.some((r) => r.status === 'running'),
|
||||
))
|
||||
const restoreDispatched = ref(false)
|
||||
const dumpsOpen = ref(false)
|
||||
const dumpsLoading = ref(false)
|
||||
const dumpsError = ref('')
|
||||
const dumps = ref([])
|
||||
const pickedDump = ref(null)
|
||||
const fetchTriggering = ref(false)
|
||||
const testSection = ref(null)
|
||||
// The notice that a restore was asked for stays up until its run is over.
|
||||
watch(runningRestore, (now, before) => {
|
||||
if (before && !now) restoreDispatched.value = false
|
||||
})
|
||||
|
||||
function restoreLine(m) {
|
||||
const n = (k) => (m?.[k] || 0).toLocaleString()
|
||||
return `${n('restored')} restored, ${n('copied')} copied from a duplicate, `
|
||||
+ `${n('present')} already there, ${n('missing')} not in the bucket, ${n('failed')} failed`
|
||||
}
|
||||
|
||||
async function openDumps() {
|
||||
dumpsLoading.value = true
|
||||
dumpsError.value = ''
|
||||
try {
|
||||
dumps.value = await store.listOffsiteDumps()
|
||||
} catch (e) {
|
||||
dumps.value = []
|
||||
dumpsError.value = e.message
|
||||
} finally {
|
||||
dumpsLoading.value = false
|
||||
dumpsOpen.value = true
|
||||
}
|
||||
}
|
||||
|
||||
function onPickDump(d) {
|
||||
pickedDump.value = d
|
||||
dumpsOpen.value = false
|
||||
confirmAction.value = 'restore'
|
||||
confirmKind.value = 'offsite'
|
||||
confirmRunId.value = ''
|
||||
confirmToken.value = d.confirm
|
||||
confirmDescription.value = (
|
||||
`Replaces this install's database with the dump taken ${formatDumpTime(d.ts)}, `
|
||||
+ 'then fetches the library\'s files from the bucket.'
|
||||
)
|
||||
confirmOpen.value = true
|
||||
}
|
||||
|
||||
async function onFetchMissing() {
|
||||
fetchTriggering.value = true
|
||||
try {
|
||||
await store.fetchMissingFiles()
|
||||
pollOnce()
|
||||
} catch {
|
||||
// store.lastError is shown above.
|
||||
} finally {
|
||||
fetchTriggering.value = false
|
||||
}
|
||||
}
|
||||
|
||||
let pollId = null
|
||||
let polls = 0
|
||||
// Coverage counts every library file, so it refreshes every sixth poll
|
||||
// (30 s) rather than with the run lists.
|
||||
const COVERAGE_EVERY = 6
|
||||
function pollOnce() {
|
||||
if (document.hidden) return
|
||||
store.loadRuns('db')
|
||||
store.loadRuns('images')
|
||||
store.loadRuns('test')
|
||||
if (offsite.value) {
|
||||
store.loadRuns('media')
|
||||
store.loadRuns('restore')
|
||||
if (polls++ % COVERAGE_EVERY === 0) {
|
||||
store.loadMediaCoverage()
|
||||
store.loadRuns('reconcile')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
store.loadSettings()
|
||||
healthStore.refresh()
|
||||
if (!store.destination) store.loadDestination().catch(() => {})
|
||||
pollOnce()
|
||||
pollId = setInterval(pollOnce, 5000)
|
||||
})
|
||||
@@ -135,11 +340,12 @@ onUnmounted(() => {
|
||||
})
|
||||
|
||||
async function onTrigger(kind) {
|
||||
const triggeringRef = kind === 'db' ? dbTriggering : imagesTriggering
|
||||
const triggeringRef = { db: dbTriggering, media: mediaTriggering }[kind]
|
||||
triggeringRef.value = true
|
||||
try {
|
||||
await store.triggerBackup(kind)
|
||||
pollOnce()
|
||||
healthStore.refresh()
|
||||
} finally {
|
||||
triggeringRef.value = false
|
||||
}
|
||||
@@ -153,9 +359,10 @@ function _openConfirm(action, run) {
|
||||
confirmAction.value = action
|
||||
confirmKind.value = run.kind
|
||||
confirmRunId.value = run.id
|
||||
confirmToken.value = ''
|
||||
confirmDescription.value = (
|
||||
action === 'restore'
|
||||
? `Source artifact: ${run.kind === 'db' ? run.sql_path : run.tar_path}`
|
||||
? `Source artifact: ${run.sql_path}`
|
||||
: 'Removes the BackupRun row and unlinks the artifact files.'
|
||||
)
|
||||
confirmOpen.value = true
|
||||
@@ -176,7 +383,10 @@ async function onTag(run) {
|
||||
|
||||
async function onConfirmSubmit(token) {
|
||||
try {
|
||||
if (confirmAction.value === 'restore') {
|
||||
if (confirmKind.value === 'offsite') {
|
||||
await store.restoreFromOffsite(pickedDump.value.key, token)
|
||||
restoreDispatched.value = true
|
||||
} else if (confirmAction.value === 'restore') {
|
||||
await store.restore(confirmRunId.value, token)
|
||||
} else {
|
||||
await store.deleteRun(confirmRunId.value, token)
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
<!--
|
||||
Off-site backup destination: an S3 bucket (Fabled Archives or any S3
|
||||
service) that backups are sent to. The five fields only work together, so
|
||||
they save as one with an explicit button rather than field by field. The
|
||||
secret key is write-only: once saved it is never shown again, and leaving the
|
||||
field blank keeps it. Test destination lists the bucket with what is typed
|
||||
in the form (saved or not) and writes nothing.
|
||||
-->
|
||||
<template>
|
||||
<MaintenanceTile
|
||||
icon="mdi-cloud-upload-outline"
|
||||
title="Off-site backup destination"
|
||||
blurb="The S3 bucket (Fabled Archives or any S3 service) that backups are sent to."
|
||||
>
|
||||
<p class="fc-muted text-body-2 mb-3">
|
||||
Backups sent here survive losing this server. FabledCurator only adds to the
|
||||
bucket and never deletes from it, so a key that may add and read objects but
|
||||
not delete them is enough, and is the safer key to use. Account logins are
|
||||
never sent.
|
||||
</p>
|
||||
|
||||
<v-alert
|
||||
v-if="loadError" type="warning" variant="tonal" density="compact" class="mb-3"
|
||||
>{{ loadError }}</v-alert>
|
||||
|
||||
<v-text-field
|
||||
v-model="form.endpoint" label="Address"
|
||||
placeholder="http://archives.example:8081" density="compact"
|
||||
hint="Include the port the S3 service listens on." persistent-hint
|
||||
class="mb-3" :disabled="busy"
|
||||
/>
|
||||
<div class="fc-dest-row mb-3">
|
||||
<v-text-field
|
||||
v-model="form.bucket" label="Bucket" density="compact" hide-details
|
||||
:disabled="busy"
|
||||
/>
|
||||
<v-text-field
|
||||
v-model="form.region" label="Region" density="compact" hide-details
|
||||
class="fc-dest-region" :disabled="busy"
|
||||
/>
|
||||
</div>
|
||||
<v-text-field
|
||||
v-model="form.access_key" label="Access key" density="compact" hide-details
|
||||
autocomplete="off" class="mb-3" :disabled="busy"
|
||||
/>
|
||||
<v-text-field
|
||||
v-model="form.secret_key" label="Secret key" type="password"
|
||||
autocomplete="new-password" density="compact"
|
||||
:placeholder="secretSet ? 'Saved. Leave blank to keep it.' : ''"
|
||||
:persistent-placeholder="secretSet"
|
||||
:hint="secretSet ? 'A saved secret is never shown again.' : ''" persistent-hint
|
||||
class="mb-3" :disabled="busy"
|
||||
/>
|
||||
<v-text-field
|
||||
v-model.number="form.upload_limit_mbps" label="Upload limit (Mbit/s)"
|
||||
type="number" min="0" density="compact" class="fc-dest-limit mb-3"
|
||||
hint="0 for no limit. A lower limit leaves more of the link free, and makes each upload take longer."
|
||||
persistent-hint :disabled="busy"
|
||||
/>
|
||||
|
||||
<div class="d-flex align-center flex-wrap mb-2" style="gap: 8px;">
|
||||
<v-btn
|
||||
size="small" variant="flat" color="accent" rounded="pill"
|
||||
prepend-icon="mdi-content-save" :loading="saving"
|
||||
:disabled="busy || !dirty" @click="onSave"
|
||||
>Save</v-btn>
|
||||
<v-btn
|
||||
size="small" variant="tonal" rounded="pill"
|
||||
prepend-icon="mdi-lan-connect" :loading="testing"
|
||||
:disabled="busy || !form.endpoint" @click="onTest"
|
||||
>Test destination</v-btn>
|
||||
<v-btn
|
||||
v-if="configured" size="small" variant="text" rounded="pill" color="error"
|
||||
:disabled="busy" @click="confirmRemove = true"
|
||||
>Remove</v-btn>
|
||||
<span v-if="savedNote" class="text-caption text-success">{{ savedNote }}</span>
|
||||
</div>
|
||||
|
||||
<v-alert
|
||||
v-if="testResult" :type="testResult.ok ? 'success' : 'error'"
|
||||
variant="tonal" density="compact" class="mb-1"
|
||||
>{{ testResult.message }}</v-alert>
|
||||
<p v-if="!configured && !testResult" class="fc-muted text-caption mb-0">
|
||||
No destination is set up, so nothing is sent off this server yet.
|
||||
</p>
|
||||
<p v-else-if="configured && !testResult" class="fc-muted text-caption mb-0">
|
||||
Database backups and the library's images and attachments go to this bucket.
|
||||
Set when they run under Backups below.
|
||||
</p>
|
||||
|
||||
<v-dialog v-model="confirmRemove" max-width="440">
|
||||
<v-card>
|
||||
<v-card-title class="text-h6">Remove the destination?</v-card-title>
|
||||
<v-card-text class="text-body-2">
|
||||
FabledCurator forgets the address and both keys and stops sending
|
||||
backups there. What was already sent stays in the bucket.
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn variant="text" @click="confirmRemove = false">Cancel</v-btn>
|
||||
<v-btn color="error" variant="flat" :loading="saving" @click="onRemove">
|
||||
Remove
|
||||
</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
</MaintenanceTile>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, onMounted, reactive, ref } from 'vue'
|
||||
|
||||
import { useBackupStore } from '../../stores/backup.js'
|
||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||
|
||||
const store = useBackupStore()
|
||||
|
||||
const FIELDS = ['endpoint', 'bucket', 'access_key', 'region']
|
||||
const DEFAULT_LIMIT = 200
|
||||
const form = reactive({
|
||||
endpoint: '', bucket: '', access_key: '', region: 'us-east-1', secret_key: '',
|
||||
upload_limit_mbps: DEFAULT_LIMIT,
|
||||
})
|
||||
const saved = ref(null)
|
||||
const loadError = ref('')
|
||||
const saving = ref(false)
|
||||
const testing = ref(false)
|
||||
const testResult = ref(null)
|
||||
const savedNote = ref('')
|
||||
const confirmRemove = ref(false)
|
||||
|
||||
const busy = computed(() => saving.value || testing.value)
|
||||
const secretSet = computed(() => Boolean(saved.value?.secret_set))
|
||||
const configured = computed(() => Boolean(saved.value?.configured))
|
||||
const dirty = computed(() => (
|
||||
!saved.value
|
||||
|| form.secret_key.trim() !== ''
|
||||
|| FIELDS.some((f) => (form[f] || '').trim() !== (saved.value[f] || ''))
|
||||
|| form.upload_limit_mbps !== saved.value.upload_limit_mbps
|
||||
))
|
||||
|
||||
function apply(dest) {
|
||||
saved.value = dest
|
||||
for (const f of FIELDS) form[f] = dest[f] || (f === 'region' ? 'us-east-1' : '')
|
||||
form.upload_limit_mbps = dest.upload_limit_mbps ?? DEFAULT_LIMIT
|
||||
form.secret_key = ''
|
||||
}
|
||||
|
||||
function fields() {
|
||||
const out = {}
|
||||
for (const f of FIELDS) out[f] = form[f]
|
||||
if (form.secret_key.trim()) out.secret_key = form.secret_key
|
||||
const limit = Number(form.upload_limit_mbps)
|
||||
if (Number.isInteger(limit) && limit >= 0) out.upload_limit_mbps = limit
|
||||
return out
|
||||
}
|
||||
|
||||
function messageOf(e) {
|
||||
return e?.body?.detail || e?.message || 'The request failed.'
|
||||
}
|
||||
|
||||
onMounted(async () => {
|
||||
try {
|
||||
apply(await store.loadDestination())
|
||||
} catch (e) {
|
||||
loadError.value = `Couldn't load the destination: ${messageOf(e)}`
|
||||
}
|
||||
})
|
||||
|
||||
async function onSave() {
|
||||
saving.value = true
|
||||
savedNote.value = ''
|
||||
testResult.value = null
|
||||
try {
|
||||
apply(await store.saveDestination(fields()))
|
||||
savedNote.value = 'Saved.'
|
||||
} catch (e) {
|
||||
testResult.value = { ok: false, message: `Not saved: ${messageOf(e)}` }
|
||||
} finally {
|
||||
saving.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onTest() {
|
||||
testing.value = true
|
||||
savedNote.value = ''
|
||||
testResult.value = null
|
||||
try {
|
||||
testResult.value = await store.testDestination(fields())
|
||||
} catch (e) {
|
||||
testResult.value = { ok: false, message: messageOf(e) }
|
||||
} finally {
|
||||
testing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onRemove() {
|
||||
saving.value = true
|
||||
try {
|
||||
apply(await store.removeDestination())
|
||||
testResult.value = null
|
||||
savedNote.value = 'Removed.'
|
||||
confirmRemove.value = false
|
||||
} catch (e) {
|
||||
testResult.value = { ok: false, message: `Not removed: ${messageOf(e)}` }
|
||||
} finally {
|
||||
saving.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-dest-row { display: flex; gap: 12px; flex-wrap: wrap; }
|
||||
.fc-dest-row > :first-child { flex: 1 1 200px; }
|
||||
.fc-dest-region { flex: 0 1 160px; }
|
||||
.fc-dest-limit { max-width: 260px; }
|
||||
</style>
|
||||
@@ -4,6 +4,7 @@
|
||||
<tr>
|
||||
<th>When</th>
|
||||
<th>Status</th>
|
||||
<th>Saved to</th>
|
||||
<th class="text-right">Duration</th>
|
||||
<th class="text-right">Size</th>
|
||||
<th>Tag</th>
|
||||
@@ -11,7 +12,8 @@
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="r in runs" :key="r.id">
|
||||
<template v-for="r in runs" :key="r.id">
|
||||
<tr>
|
||||
<td class="fc-tabular" :title="r.started_at">
|
||||
{{ formatRelative(r.started_at) }}
|
||||
</td>
|
||||
@@ -20,6 +22,21 @@
|
||||
{{ statusIcon(r.status) }}
|
||||
</v-icon>
|
||||
{{ r.status }}
|
||||
<div
|
||||
v-if="r.error" class="fc-run-error text-caption text-error" :title="r.error"
|
||||
>{{ r.error }}</div>
|
||||
<v-btn
|
||||
v-if="failures(r).length"
|
||||
size="x-small" variant="text" color="error" class="px-1"
|
||||
:append-icon="open[r.id] ? 'mdi-chevron-up' : 'mdi-chevron-down'"
|
||||
@click="open[r.id] = !open[r.id]"
|
||||
>{{ failedLabel(r) }}</v-btn>
|
||||
</td>
|
||||
<td :title="r.offsite ? `${r.offsite.endpoint} · ${r.offsite.key}` : ''">
|
||||
<template v-if="r.offsite">
|
||||
<v-icon size="small" class="mr-1">mdi-cloud-check-outline</v-icon>{{ r.offsite.bucket }}
|
||||
</template>
|
||||
<span v-else class="fc-muted">This server</span>
|
||||
</td>
|
||||
<td class="text-right fc-tabular">{{ formatDuration(r) }}</td>
|
||||
<td class="text-right fc-tabular">{{ formatBytes(r.size_bytes) }}</td>
|
||||
@@ -30,7 +47,8 @@
|
||||
<span v-else class="fc-muted">—</span>
|
||||
</td>
|
||||
<td class="text-right">
|
||||
<KebabMenu :label="`Actions for backup ${r.id}`">
|
||||
<span v-if="r.offsite || !actions" class="fc-muted">—</span>
|
||||
<KebabMenu v-else :label="`Actions for backup ${r.id}`">
|
||||
<v-list-item
|
||||
:title="r.tag ? 'Untag' : 'Tag…'"
|
||||
@click="$emit('tag', r)"
|
||||
@@ -39,6 +57,11 @@
|
||||
title="Restore…" :disabled="r.status !== 'ok'"
|
||||
@click="$emit('restore', r)"
|
||||
/>
|
||||
<v-list-item
|
||||
v-if="r.kind === 'db'"
|
||||
title="Test restore" :disabled="r.status !== 'ok'"
|
||||
@click="$emit('test', r)"
|
||||
/>
|
||||
<v-list-item
|
||||
title="Delete…"
|
||||
@click="$emit('delete', r)"
|
||||
@@ -46,19 +69,60 @@
|
||||
</KebabMenu>
|
||||
</td>
|
||||
</tr>
|
||||
<!-- The files a run couldn't send or fetch, each with what went wrong.
|
||||
The run keeps the first few; the count covers them all. -->
|
||||
<tr v-if="open[r.id]" class="fc-failures">
|
||||
<td colspan="7">
|
||||
<ul class="fc-failures__list text-caption">
|
||||
<li v-for="(f, i) in failures(r)" :key="i">
|
||||
<code :title="f.path">{{ f.path }}</code>
|
||||
<span class="text-error"> — {{ f.error }}</span>
|
||||
</li>
|
||||
</ul>
|
||||
<div v-if="moreFailed(r)" class="text-caption fc-muted">
|
||||
and {{ moreFailed(r).toLocaleString() }} more.
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
</template>
|
||||
<tr v-if="!runs.length">
|
||||
<td colspan="6" class="text-center fc-muted py-4">No backups yet.</td>
|
||||
<td colspan="7" class="text-center fc-muted py-4">{{ emptyText }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</v-table>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { reactive } from 'vue'
|
||||
|
||||
import { formatRelative as fmtRelative } from '../../utils/date.js'
|
||||
import KebabMenu from '../common/KebabMenu.vue'
|
||||
|
||||
defineProps({ runs: { type: Array, default: () => [] } })
|
||||
defineEmits(['restore', 'delete', 'tag'])
|
||||
defineProps({
|
||||
runs: { type: Array, default: () => [] },
|
||||
emptyText: { type: String, default: 'No backups yet.' },
|
||||
// False for runs that are records, not backups (tests): no Tag, Restore
|
||||
// or Delete menu.
|
||||
actions: { type: Boolean, default: true },
|
||||
})
|
||||
defineEmits(['restore', 'delete', 'tag', 'test'])
|
||||
|
||||
// Which runs have their failed-file list open.
|
||||
const open = reactive({})
|
||||
|
||||
function failures(r) {
|
||||
return r.manifest?.failures || []
|
||||
}
|
||||
function failedCount(r) {
|
||||
return r.manifest?.failed || failures(r).length
|
||||
}
|
||||
function moreFailed(r) {
|
||||
return Math.max(0, failedCount(r) - failures(r).length)
|
||||
}
|
||||
function failedLabel(r) {
|
||||
const n = failedCount(r)
|
||||
return `${n.toLocaleString()} file${n === 1 ? '' : 's'} failed`
|
||||
}
|
||||
|
||||
function statusIcon(s) {
|
||||
return {
|
||||
@@ -97,4 +161,13 @@ function formatRelative(iso) {
|
||||
<style scoped>
|
||||
.fc-backup-table { background: transparent; }
|
||||
.fc-tabular { font-variant-numeric: tabular-nums; }
|
||||
.fc-failures td { border-top: 0 !important; }
|
||||
.fc-failures__list { margin: 0 0 6px; padding-left: 18px; }
|
||||
.fc-failures__list code {
|
||||
display: inline-block; max-width: 420px; overflow: hidden;
|
||||
text-overflow: ellipsis; white-space: nowrap; vertical-align: bottom;
|
||||
}
|
||||
.fc-run-error {
|
||||
max-width: 320px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
<template>
|
||||
<div>
|
||||
<h3 class="fc-section-title">Test a backup</h3>
|
||||
<p class="fc-muted text-body-2 mb-2">
|
||||
Load a database dump into a separate, temporary database, check that every
|
||||
table came back with the rows it held when the dump was taken, and bring it
|
||||
up to this version. The temporary database is then deleted. This install's
|
||||
database and library aren't touched.
|
||||
<template v-if="store.databaseBytes">
|
||||
It needs about {{ humanBytes(store.databaseBytes) }} free on the
|
||||
database's disk while it runs.
|
||||
</template>
|
||||
</p>
|
||||
<div class="fc-settings-row mb-3">
|
||||
<v-btn
|
||||
color="accent" variant="tonal" rounded="pill"
|
||||
prepend-icon="mdi-database-check-outline"
|
||||
:disabled="blocked || (!bucket && !newestLocal)" :loading="starting"
|
||||
@click="startNewest"
|
||||
>Test the newest dump</v-btn>
|
||||
<v-btn
|
||||
v-if="bucket"
|
||||
variant="text" rounded="pill"
|
||||
:disabled="blocked" :loading="dumpsLoading"
|
||||
@click="openDumps"
|
||||
>Choose a dump…</v-btn>
|
||||
</div>
|
||||
|
||||
<v-alert
|
||||
v-if="running || dispatched"
|
||||
type="info" variant="tonal" density="compact" class="mb-3"
|
||||
>
|
||||
Testing the dump<template v-if="running?.manifest?.dump?.ts">
|
||||
taken {{ formatDumpTime(running.manifest.dump.ts) }}</template>:
|
||||
{{ phaseText }}.
|
||||
</v-alert>
|
||||
|
||||
<!-- The newest finished test: its verdict, and every table on request. -->
|
||||
<div v-if="last" class="mb-3">
|
||||
<div class="d-flex align-start text-body-2">
|
||||
<v-icon
|
||||
size="small" class="mr-2 mt-1"
|
||||
:color="last.status === 'ok' ? 'success' : 'error'"
|
||||
>{{ last.status === 'ok' ? 'mdi-check-circle' : 'mdi-close-circle' }}</v-icon>
|
||||
<div>
|
||||
<strong>Last test</strong>
|
||||
({{ formatRelative(last.finished_at) }}<template v-if="last.manifest?.dump?.ts">,
|
||||
dump taken {{ formatDumpTime(last.manifest.dump.ts) }}</template>):
|
||||
{{ last.manifest?.summary || last.error }}
|
||||
<div v-if="upgradeLine" class="text-caption fc-muted">{{ upgradeLine }}</div>
|
||||
</div>
|
||||
</div>
|
||||
<v-btn
|
||||
v-if="tables.length"
|
||||
size="x-small" variant="text" class="mt-1"
|
||||
:append-icon="showTables ? 'mdi-chevron-up' : 'mdi-chevron-down'"
|
||||
@click="showTables = !showTables"
|
||||
>{{ showTables ? 'Hide' : 'Show' }} each table</v-btn>
|
||||
<v-table v-if="showTables" density="compact" class="fc-test-tables mt-1">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Table</th>
|
||||
<th class="text-right">In the dump</th>
|
||||
<th class="text-right">Restored</th>
|
||||
<th class="text-right">Live now</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr
|
||||
v-for="t in tables" :key="t.table"
|
||||
:class="{ 'text-error': mismatched.has(t.table) }"
|
||||
>
|
||||
<td><code>{{ t.table }}</code></td>
|
||||
<td class="text-right">{{ num(t.dump) }}</td>
|
||||
<td class="text-right">{{ num(t.restored) }}</td>
|
||||
<td class="text-right fc-muted">{{ num(t.live) }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</v-table>
|
||||
</div>
|
||||
|
||||
<BackupRunsTable :runs="store.testRuns" :actions="false" empty-text="No tests yet." />
|
||||
|
||||
<DumpPickerDialog
|
||||
v-if="bucket"
|
||||
v-model="dumpsOpen" action="test"
|
||||
:bucket="bucket" :dumps="dumps" :error="dumpsError"
|
||||
@pick="onPick"
|
||||
/>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, ref, watch } from 'vue'
|
||||
|
||||
import { useBackupStore } from '../../stores/backup.js'
|
||||
import { humanBytes } from '../../utils/bytes.js'
|
||||
import { formatDumpTime, formatRelative } from '../../utils/date.js'
|
||||
import BackupRunsTable from './BackupRunsTable.vue'
|
||||
import DumpPickerDialog from './DumpPickerDialog.vue'
|
||||
|
||||
const props = defineProps({
|
||||
// The off-site bucket's name when one is set up; otherwise the dumps on
|
||||
// this server are tested.
|
||||
bucket: { type: String, default: '' },
|
||||
})
|
||||
const emit = defineEmits(['started'])
|
||||
|
||||
const store = useBackupStore()
|
||||
|
||||
const running = computed(() => store.testRuns.find((r) => r.status === 'running'))
|
||||
const last = computed(() => store.testRuns.find((r) => r.status !== 'running'))
|
||||
const restoring = computed(() => store.restoreRuns.some((r) => r.status === 'restoring'))
|
||||
const blocked = computed(() => Boolean(running.value || restoring.value))
|
||||
const newestLocal = computed(() => store.dbRuns.find((r) => r.status === 'ok' && !r.offsite))
|
||||
|
||||
const PHASES = {
|
||||
fetching: 'downloading it from the bucket',
|
||||
loading: 'loading it into a temporary database',
|
||||
comparing: 'counting every table',
|
||||
upgrading: 'bringing it up to this version',
|
||||
}
|
||||
const phaseText = computed(() => PHASES[running.value?.manifest?.phase] || 'starting')
|
||||
|
||||
const tables = computed(() => last.value?.manifest?.tables || [])
|
||||
const mismatched = computed(() => new Set(last.value?.manifest?.mismatched || []))
|
||||
const showTables = ref(false)
|
||||
const upgradeLine = computed(() => {
|
||||
const m = last.value?.manifest
|
||||
if (!m?.alembic_now) return ''
|
||||
return m.alembic_dump === m.alembic_now
|
||||
? 'Its schema is already this version\'s.'
|
||||
: `Its schema was brought up from ${m.alembic_dump || 'none'} to ${m.alembic_now}.`
|
||||
})
|
||||
|
||||
function num(n) {
|
||||
return n == null ? '—' : n.toLocaleString()
|
||||
}
|
||||
|
||||
// "Testing…" shows from the click until the run appears.
|
||||
const dispatched = ref(false)
|
||||
watch(running, (now) => { if (now) dispatched.value = false })
|
||||
|
||||
const starting = ref(false)
|
||||
async function start(what) {
|
||||
starting.value = true
|
||||
try {
|
||||
await store.testDump(what)
|
||||
dispatched.value = true
|
||||
emit('started')
|
||||
} catch {
|
||||
// store.lastError is shown on the card.
|
||||
} finally {
|
||||
starting.value = false
|
||||
}
|
||||
}
|
||||
function startNewest() {
|
||||
start(props.bucket ? {} : { runId: newestLocal.value.id })
|
||||
}
|
||||
|
||||
const dumpsOpen = ref(false)
|
||||
const dumpsLoading = ref(false)
|
||||
const dumpsError = ref('')
|
||||
const dumps = ref([])
|
||||
async function openDumps() {
|
||||
dumpsLoading.value = true
|
||||
dumpsError.value = ''
|
||||
try {
|
||||
dumps.value = await store.listOffsiteDumps()
|
||||
} catch (e) {
|
||||
dumps.value = []
|
||||
dumpsError.value = e.message
|
||||
} finally {
|
||||
dumpsLoading.value = false
|
||||
dumpsOpen.value = true
|
||||
}
|
||||
}
|
||||
function onPick(d) {
|
||||
dumpsOpen.value = false
|
||||
start({ key: d.key })
|
||||
}
|
||||
|
||||
defineExpose({ start })
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-section-title {
|
||||
font-family: 'Fraunces', Georgia, serif;
|
||||
font-size: 16px; font-weight: 500;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
.fc-settings-row {
|
||||
display: flex; gap: 12px; align-items: center; flex-wrap: wrap;
|
||||
}
|
||||
.fc-test-tables { background: transparent; font-variant-numeric: tabular-nums; }
|
||||
</style>
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user