Compare commits
69 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 667b05f14e | |||
| 8de7ccd07d | |||
| d65f0b2091 | |||
| 856e9104b4 | |||
| 66f19d67f5 | |||
| 6fc8ae3106 | |||
| 0397642b21 | |||
| a5101494b6 | |||
| e3a7aff7a3 | |||
| 9cd6d09e60 | |||
| 237575447d | |||
| 810baf63ac | |||
| 44bb12a93d | |||
| 1eefed9ab3 | |||
| adeee64a2d | |||
| ed358757dc | |||
| 99b66aa85f | |||
| 77f7a23410 | |||
| d181f4afb8 | |||
| ff9e96e0e2 | |||
| 61ce1ce13c | |||
| d28db32012 | |||
| 77e9859da3 | |||
| 2886fa4997 | |||
| 36f8ec80fd | |||
| f256f587ee | |||
| e35fb1edf7 | |||
| 08420cd619 | |||
| 8649a13118 | |||
| 8979e0e377 | |||
| 00e2608ba1 | |||
| 9ab5d709c8 | |||
| 44410db492 | |||
| e76aa36a29 | |||
| c95b760294 | |||
| 35fe420701 | |||
| 9e74c80e2f | |||
| c87e8e0932 | |||
| 8c3900b998 | |||
| 972d9014ce | |||
| 75b6b8056e | |||
| 42ddac9996 | |||
| 384d8d5e50 | |||
| 1322056b22 | |||
| 32bdde049f | |||
| 9d18dacbe8 | |||
| 171c486939 | |||
| 21c1b0a81c | |||
| 597c6d48d3 | |||
| a37dad33c7 | |||
| cabd73287a | |||
| def967a1a8 | |||
| eebc8e2413 | |||
| 2358cedf3e | |||
| 215a8993a1 | |||
| a459d21a65 | |||
| 73520b7cc3 | |||
| 56970fb66d | |||
| bf8eb4468f | |||
| e1fc65bd1b | |||
| 104cac5dca | |||
| 319e8c1d18 | |||
| dcfe55d731 | |||
| e3cdd0f92b | |||
| e77afe8295 | |||
| 57a22d6098 | |||
| a85880f965 | |||
| 407de18ff6 | |||
| b1b129ce9f |
@@ -242,20 +242,28 @@ jobs:
|
|||||||
id: tag
|
id: tag
|
||||||
run: |
|
run: |
|
||||||
# Three trigger shapes:
|
# Three trigger shapes:
|
||||||
# refs/tags/v… → tag-push: publish ONLY the immutable version
|
# refs/tags/v… → tag-push: opt-in milestone label (vYY.MM.DD,
|
||||||
# tag (e.g. :v26.05.26.5). Don't touch :latest;
|
# no `.N` per family release-posture rule).
|
||||||
# that already got published by the main-push
|
# Publish ONLY the immutable version tag;
|
||||||
# build for the merge commit.
|
# don't touch :latest (the main-push build
|
||||||
# refs/heads/main → push to main (incl. PR merge commits):
|
# for the merge commit already did that).
|
||||||
# publish :main + :latest (floating).
|
# refs/heads/main → push to main: publish :main + :latest
|
||||||
|
# (floating) AND :c-<short_sha> (immutable
|
||||||
|
# per-commit rollback substrate, per family
|
||||||
|
# release-posture rule "Tags are milestones,
|
||||||
|
# not gates — commit-SHA images are the
|
||||||
|
# rollback unit"). Rollback to any commit
|
||||||
|
# becomes `docker pull …:c-<sha>` without a
|
||||||
|
# release ceremony.
|
||||||
# anything else → safety net; shouldn't fire given the `on:`
|
# anything else → safety net; shouldn't fire given the `on:`
|
||||||
# config above (dev was dropped). Tag :dev to
|
# config above. Tag :dev to surface the
|
||||||
# surface the unexpected run in the registry.
|
# unexpected run in the registry.
|
||||||
|
SHORT_SHA="${GITHUB_SHA:0:7}"
|
||||||
if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then
|
if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then
|
||||||
TAG_NAME="${GITHUB_REF#refs/tags/}"
|
TAG_NAME="${GITHUB_REF#refs/tags/}"
|
||||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:${TAG_NAME}" >> "$GITHUB_OUTPUT"
|
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:${TAG_NAME}" >> "$GITHUB_OUTPUT"
|
||||||
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:main,git.fabledsword.com/bvandeusen/fabledcurator:latest" >> "$GITHUB_OUTPUT"
|
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:main,git.fabledsword.com/bvandeusen/fabledcurator:latest,git.fabledsword.com/bvandeusen/fabledcurator:c-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
|
||||||
else
|
else
|
||||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:dev" >> "$GITHUB_OUTPUT"
|
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:dev" >> "$GITHUB_OUTPUT"
|
||||||
fi
|
fi
|
||||||
@@ -286,13 +294,15 @@ jobs:
|
|||||||
id: tag
|
id: tag
|
||||||
run: |
|
run: |
|
||||||
# Mirrors build-web's three-shape logic (tag-push / main-push /
|
# Mirrors build-web's three-shape logic (tag-push / main-push /
|
||||||
# safety-net dev). The -ml image follows the same release cadence
|
# safety-net dev) including the per-commit :c-<short_sha> tag
|
||||||
# as the web image.
|
# on main-push per the family release-posture rule. The -ml
|
||||||
|
# image follows the same release cadence as the web image.
|
||||||
|
SHORT_SHA="${GITHUB_SHA:0:7}"
|
||||||
if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then
|
if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then
|
||||||
TAG_NAME="${GITHUB_REF#refs/tags/}"
|
TAG_NAME="${GITHUB_REF#refs/tags/}"
|
||||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:${TAG_NAME}" >> "$GITHUB_OUTPUT"
|
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:${TAG_NAME}" >> "$GITHUB_OUTPUT"
|
||||||
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:main,git.fabledsword.com/bvandeusen/fabledcurator-ml:latest" >> "$GITHUB_OUTPUT"
|
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:main,git.fabledsword.com/bvandeusen/fabledcurator-ml:latest,git.fabledsword.com/bvandeusen/fabledcurator-ml:c-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
|
||||||
else
|
else
|
||||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:dev" >> "$GITHUB_OUTPUT"
|
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:dev" >> "$GITHUB_OUTPUT"
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
name: CI
|
name: CI
|
||||||
|
|
||||||
# CI lanes per FabledRulebook/forgejo.md "CI philosophy":
|
# CI lanes per FabledRulebook/forgejo.md "CI philosophy":
|
||||||
# - backend-lint-and-test: ruff + `pytest -m "not integration"`, no service containers.
|
# - lint: ruff only, no dep install — fast-fail for the common lint bounce.
|
||||||
|
# - backend-lint-and-test: `pytest -m "not integration"`, no service containers.
|
||||||
# - frontend-build: vitest unit + vite build.
|
# - frontend-build: vitest unit + vite build.
|
||||||
# - integration: pgvector + redis service containers; alembic + `pytest -m integration`.
|
# - integration: pgvector + redis service containers; alembic + `pytest -m integration`.
|
||||||
|
|
||||||
@@ -14,6 +15,20 @@ on:
|
|||||||
# (single-operator Forgejo repo) so push coverage is complete.
|
# (single-operator Forgejo repo) so push coverage is complete.
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
|
# Fast-fail lint lane. ruff is pre-installed in the ci-python image, so
|
||||||
|
# this runs with NO dependency install and surfaces the most common bounce
|
||||||
|
# class (lint: I001 / UP037 / ASYNC109 / W293 …) in seconds — instead of
|
||||||
|
# after the backend job's ~30-60s wheel install. ruff is static analysis,
|
||||||
|
# so no DB/secret env is needed.
|
||||||
|
lint:
|
||||||
|
runs-on: python-ci
|
||||||
|
container:
|
||||||
|
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- name: Ruff lint
|
||||||
|
run: ruff check backend/ tests/ alembic/
|
||||||
|
|
||||||
backend-lint-and-test:
|
backend-lint-and-test:
|
||||||
runs-on: python-ci
|
runs-on: python-ci
|
||||||
container:
|
container:
|
||||||
@@ -51,9 +66,8 @@ jobs:
|
|||||||
pip install -r requirements.txt pytest pytest-asyncio
|
pip install -r requirements.txt pytest pytest-asyncio
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Ruff lint
|
# Ruff moved to the dedicated fast `lint` job above (fails in seconds,
|
||||||
run: ruff check backend/ tests/ alembic/
|
# no dep install). This job is now unit tests only.
|
||||||
|
|
||||||
- name: Pytest (unit only — integration runs in the integration job)
|
- name: Pytest (unit only — integration runs in the integration job)
|
||||||
run: pytest tests/ -v -m "not integration"
|
run: pytest tests/ -v -m "not integration"
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,53 @@
|
|||||||
|
"""import_task.recovery_count + refetched — poison-pill circuit breaker
|
||||||
|
|
||||||
|
Revision ID: 0026
|
||||||
|
Revises: 0025
|
||||||
|
Create Date: 2026-05-28
|
||||||
|
|
||||||
|
Backs the import-task resilience work (operator-flagged 2026-05-28):
|
||||||
|
|
||||||
|
- recovery_count: how many times recover_interrupted_tasks has
|
||||||
|
re-queued this row from a stuck 'processing' state. A row that
|
||||||
|
hard-crashes the worker (OOM / segfault on a corrupt or oversized
|
||||||
|
input) leaves no terminal flip, so the sweep re-queues it — and
|
||||||
|
without a cap it would loop forever, re-crashing the worker each
|
||||||
|
time. After MAX_RECOVERY_ATTEMPTS the sweep marks it 'failed' with a
|
||||||
|
diagnostic instead.
|
||||||
|
|
||||||
|
- refetched: whether a one-shot re-download has already been attempted
|
||||||
|
for this task's file. Bounds the Layer-2 re-fetch remediation to a
|
||||||
|
single attempt so source-side corruption doesn't loop.
|
||||||
|
|
||||||
|
Both default to 0 / false; additive, no backfill needed.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0026"
|
||||||
|
down_revision: Union[str, None] = "0025"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"import_task",
|
||||||
|
sa.Column(
|
||||||
|
"recovery_count", sa.Integer(), nullable=False,
|
||||||
|
server_default="0",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"import_task",
|
||||||
|
sa.Column(
|
||||||
|
"refetched", sa.Boolean(), nullable=False,
|
||||||
|
server_default=sa.false(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("import_task", "refetched")
|
||||||
|
op.drop_column("import_task", "recovery_count")
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
"""drop migration_run — one-and-done GS/IR migration tooling removed
|
||||||
|
|
||||||
|
Revision ID: 0027
|
||||||
|
Revises: 0026
|
||||||
|
Create Date: 2026-05-29
|
||||||
|
|
||||||
|
The GS/IR migration tooling (services/migrators, /api/migrate, the
|
||||||
|
run_migration task, LegacyMigrationCard, and the MigrationRun model) was
|
||||||
|
removed after the migration cutover completed. This drops its now-orphaned
|
||||||
|
run-log table. Downgrade recreates the table (mirrors the old model) so the
|
||||||
|
migration is reversible.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
|
|
||||||
|
revision: str = "0027"
|
||||||
|
down_revision: Union[str, None] = "0026"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.drop_table("migration_run")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"migration_run",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column("kind", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("dry_run", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||||
|
sa.Column(
|
||||||
|
"started_at", sa.DateTime(timezone=True), nullable=False,
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"counts", JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb"),
|
||||||
|
),
|
||||||
|
sa.Column("error", sa.Text(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"metadata", JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index("ix_migration_run_kind", "migration_run", ["kind"])
|
||||||
|
op.create_index("ix_migration_run_status", "migration_run", ["status"])
|
||||||
@@ -0,0 +1,190 @@
|
|||||||
|
"""collapse-sidecar-synthetic: repoint Posts/ImageProvenance/DownloadEvents
|
||||||
|
from `sidecar:<platform>:<slug>` synthetic Source anchors onto the real
|
||||||
|
Source for the same (artist, platform) when one exists, then delete the
|
||||||
|
synthetic.
|
||||||
|
|
||||||
|
Revision ID: 0028
|
||||||
|
Revises: 0027
|
||||||
|
Create Date: 2026-05-31
|
||||||
|
|
||||||
|
Background: alembic 0022 (2026-05-26) consolidated the old per-post-URL
|
||||||
|
Source rows into one canonical Source per (artist, platform). When NO
|
||||||
|
real campaign URL was salvageable among the candidates, it rewrote the
|
||||||
|
canonical row to url='sidecar:<platform>:<slug>' enabled=false as a
|
||||||
|
disabled anchor for any Posts already attached.
|
||||||
|
|
||||||
|
That was fine while it was the only Source for that artist+platform.
|
||||||
|
But: the unique constraint on Source is (artist_id, platform, url), not
|
||||||
|
(artist_id, platform). When the operator later added the real
|
||||||
|
subscription via the UI / extension / etc., a SECOND row landed —
|
||||||
|
the real one — with id > the synthetic. Both coexisted.
|
||||||
|
|
||||||
|
Two follow-on problems surfaced 2026-05-31:
|
||||||
|
|
||||||
|
1. The Subscriptions UI listed both rows. The synthetic was disabled
|
||||||
|
so the scheduler never polled it, but it looked like a phantom
|
||||||
|
subscription. (Fixed in same commit by SourceService.list filter.)
|
||||||
|
2. importer._source_for_sidecar picked Source by `ORDER BY id ASC
|
||||||
|
LIMIT 1`, so EVERY gallery-dl download since the real Source was
|
||||||
|
added attached its Post to the SYNTHETIC anchor, not the real
|
||||||
|
Source. (Fixed in same commit by preferring non-sidecar URLs.)
|
||||||
|
|
||||||
|
This migration is the data half of the cleanup: for every (artist,
|
||||||
|
platform) with both a synthetic AND a real Source, repoint the
|
||||||
|
synthetic's children (Posts, ImageProvenance, DownloadEvents) onto the
|
||||||
|
real Source and delete the synthetic. Reuses the same epid/provenance
|
||||||
|
collision dance from alembic 0022 because the same uniqueness
|
||||||
|
constraints fire row-by-row during bulk UPDATEs.
|
||||||
|
|
||||||
|
Lone synthetic anchors — those where no real Source for the same
|
||||||
|
(artist, platform) exists (e.g., filesystem-imported artist with no
|
||||||
|
subscription added) — are LEFT INTACT. They anchor real imported
|
||||||
|
content; deleting them would CASCADE-delete the Posts the operator
|
||||||
|
imported. The SourceService.list filter hides them from the UI; the
|
||||||
|
operator can delete them by hand if they want the underlying imports
|
||||||
|
gone.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
revision: str = "0028"
|
||||||
|
down_revision: Union[str, None] = "0027"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
conn = op.get_bind()
|
||||||
|
|
||||||
|
# Find (artist_id, platform) groups where BOTH a sidecar synthetic
|
||||||
|
# and at least one real Source exist.
|
||||||
|
groups = conn.execute(text("""
|
||||||
|
SELECT artist_id, platform
|
||||||
|
FROM source
|
||||||
|
GROUP BY artist_id, platform
|
||||||
|
HAVING bool_or(url LIKE 'sidecar:%')
|
||||||
|
AND bool_or(url NOT LIKE 'sidecar:%')
|
||||||
|
""")).fetchall()
|
||||||
|
|
||||||
|
for artist_id, platform in groups:
|
||||||
|
rows = conn.execute(
|
||||||
|
text("""
|
||||||
|
SELECT id, url FROM source
|
||||||
|
WHERE artist_id = :a AND platform = :p
|
||||||
|
ORDER BY id ASC
|
||||||
|
"""),
|
||||||
|
{"a": artist_id, "p": platform},
|
||||||
|
).fetchall()
|
||||||
|
|
||||||
|
synthetic_ids = [sid for sid, url in rows if url.startswith("sidecar:")]
|
||||||
|
real_rows = [(sid, url) for sid, url in rows if not url.startswith("sidecar:")]
|
||||||
|
if not synthetic_ids or not real_rows:
|
||||||
|
continue # belt+suspenders; the GROUP BY already filtered
|
||||||
|
|
||||||
|
# Canonical real: lowest-id non-sidecar Source.
|
||||||
|
canonical_id = real_rows[0][0]
|
||||||
|
|
||||||
|
# STEP A: PRE-merge Post collisions on (canonical, external_post_id).
|
||||||
|
# Mirror alembic 0022's pre-merge logic — when synth has Post X
|
||||||
|
# epid=N and real has Post Y epid=N, the bulk UPDATE below would
|
||||||
|
# trip uq_post_source_external_id row-by-row. Group all Posts
|
||||||
|
# under (canonical + synthetics) by epid; for any group >1,
|
||||||
|
# pick a keep (prefer one already under canonical, else lowest
|
||||||
|
# id) and merge the rest into it.
|
||||||
|
all_posts = conn.execute(
|
||||||
|
text("""
|
||||||
|
SELECT external_post_id, id, source_id
|
||||||
|
FROM post
|
||||||
|
WHERE source_id = :canonical OR source_id = ANY(:synths)
|
||||||
|
ORDER BY external_post_id, id
|
||||||
|
"""),
|
||||||
|
{"canonical": canonical_id, "synths": synthetic_ids},
|
||||||
|
).fetchall()
|
||||||
|
by_epid: dict = {}
|
||||||
|
for epid, post_id, src_id in all_posts:
|
||||||
|
by_epid.setdefault(epid, []).append((post_id, src_id))
|
||||||
|
for _epid, posts in by_epid.items():
|
||||||
|
if len(posts) <= 1:
|
||||||
|
continue
|
||||||
|
canonical_side = [p for p in posts if p[1] == canonical_id]
|
||||||
|
keep_id = canonical_side[0][0] if canonical_side else posts[0][0]
|
||||||
|
drop_ids = [p[0] for p in posts if p[0] != keep_id]
|
||||||
|
for drop_id in drop_ids:
|
||||||
|
# Pre-delete image_provenance rows under drop_ whose
|
||||||
|
# image_record_id already has provenance under keep —
|
||||||
|
# avoids tripping uq_image_provenance_image_post (0021)
|
||||||
|
# row-by-row during the repoint UPDATE.
|
||||||
|
conn.execute(
|
||||||
|
text("""
|
||||||
|
DELETE FROM image_provenance
|
||||||
|
WHERE post_id = :drop_
|
||||||
|
AND image_record_id IN (
|
||||||
|
SELECT image_record_id FROM image_provenance
|
||||||
|
WHERE post_id = :keep
|
||||||
|
)
|
||||||
|
"""),
|
||||||
|
{"keep": keep_id, "drop_": drop_id},
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
text("""
|
||||||
|
UPDATE image_provenance SET post_id = :keep
|
||||||
|
WHERE post_id = :drop_
|
||||||
|
"""),
|
||||||
|
{"keep": keep_id, "drop_": drop_id},
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
text("""
|
||||||
|
UPDATE image_record SET primary_post_id = :keep
|
||||||
|
WHERE primary_post_id = :drop_
|
||||||
|
"""),
|
||||||
|
{"keep": keep_id, "drop_": drop_id},
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
text("DELETE FROM post WHERE id = :drop_"),
|
||||||
|
{"drop_": drop_id},
|
||||||
|
)
|
||||||
|
|
||||||
|
# STEP B: Bulk reparent the remaining Posts off the synthetics.
|
||||||
|
conn.execute(
|
||||||
|
text("""
|
||||||
|
UPDATE post SET source_id = :canonical
|
||||||
|
WHERE source_id = ANY(:synths)
|
||||||
|
"""),
|
||||||
|
{"canonical": canonical_id, "synths": synthetic_ids},
|
||||||
|
)
|
||||||
|
|
||||||
|
# STEP C: Reparent ImageProvenance.source_id (denormalized FK;
|
||||||
|
# no UNIQUE on source_id, safe bulk).
|
||||||
|
conn.execute(
|
||||||
|
text("""
|
||||||
|
UPDATE image_provenance SET source_id = :canonical
|
||||||
|
WHERE source_id = ANY(:synths)
|
||||||
|
"""),
|
||||||
|
{"canonical": canonical_id, "synths": synthetic_ids},
|
||||||
|
)
|
||||||
|
|
||||||
|
# STEP D: Reparent any DownloadEvent.source_id. Synthetics are
|
||||||
|
# enabled=false so the scheduler never created events for them;
|
||||||
|
# this is belt+suspenders for any rows planted by manual force
|
||||||
|
# or older code paths.
|
||||||
|
conn.execute(
|
||||||
|
text("""
|
||||||
|
UPDATE download_event SET source_id = :canonical
|
||||||
|
WHERE source_id = ANY(:synths)
|
||||||
|
"""),
|
||||||
|
{"canonical": canonical_id, "synths": synthetic_ids},
|
||||||
|
)
|
||||||
|
|
||||||
|
# STEP E: Drop the now-empty synthetics.
|
||||||
|
conn.execute(
|
||||||
|
text("DELETE FROM source WHERE id = ANY(:synths)"),
|
||||||
|
{"synths": synthetic_ids},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
# Lossy migration — synthetic Sources deleted, Posts repointed and
|
||||||
|
# potentially merged. No safe downgrade.
|
||||||
|
pass
|
||||||
@@ -33,12 +33,6 @@ def create_app() -> Quart:
|
|||||||
|
|
||||||
app = Quart(__name__)
|
app = Quart(__name__)
|
||||||
app.secret_key = cfg.secret_key
|
app.secret_key = cfg.secret_key
|
||||||
# FC-5: legacy IR ingest JSON can run to tens of MB (hundreds of
|
|
||||||
# thousands of image_tag_associations). Werkzeug's default form
|
|
||||||
# memory cap is 500KB; raise both ceilings so the multipart upload
|
|
||||||
# for /api/migrate/ir_ingest doesn't 413.
|
|
||||||
app.config["MAX_CONTENT_LENGTH"] = 1024 * 1024 * 1024 # 1 GB
|
|
||||||
app.config["MAX_FORM_MEMORY_SIZE"] = 1024 * 1024 * 1024 # 1 GB
|
|
||||||
|
|
||||||
for bp in all_blueprints():
|
for bp in all_blueprints():
|
||||||
app.register_blueprint(bp)
|
app.register_blueprint(bp)
|
||||||
|
|||||||
@@ -26,7 +26,6 @@ def all_blueprints() -> list[Blueprint]:
|
|||||||
from .extension import extension_bp
|
from .extension import extension_bp
|
||||||
from .gallery import gallery_bp
|
from .gallery import gallery_bp
|
||||||
from .import_admin import import_admin_bp
|
from .import_admin import import_admin_bp
|
||||||
from .migrate import migrate_bp
|
|
||||||
from .ml_admin import ml_admin_bp
|
from .ml_admin import ml_admin_bp
|
||||||
from .platforms import platforms_bp
|
from .platforms import platforms_bp
|
||||||
from .posts import posts_bp
|
from .posts import posts_bp
|
||||||
@@ -54,7 +53,6 @@ def all_blueprints() -> list[Blueprint]:
|
|||||||
admin_bp,
|
admin_bp,
|
||||||
cleanup_bp,
|
cleanup_bp,
|
||||||
import_admin_bp,
|
import_admin_bp,
|
||||||
migrate_bp,
|
|
||||||
suggestions_bp,
|
suggestions_bp,
|
||||||
allowlist_bp,
|
allowlist_bp,
|
||||||
aliases_bp,
|
aliases_bp,
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
"""Shared API response helpers."""
|
||||||
|
|
||||||
|
from quart import jsonify
|
||||||
|
|
||||||
|
|
||||||
|
def error_response(
|
||||||
|
error: str, *, status: int = 400, detail: str | None = None, **extra,
|
||||||
|
):
|
||||||
|
"""JSON error body + HTTP status. `detail` is included only when given;
|
||||||
|
`extra` keys are merged into the body. Returns the (response, status)
|
||||||
|
tuple Quart expects. Imported as `_bad` by the blueprints."""
|
||||||
|
body = {"error": error}
|
||||||
|
if detail is not None:
|
||||||
|
body["detail"] = detail
|
||||||
|
body.update(extra)
|
||||||
|
return jsonify(body), status
|
||||||
@@ -6,6 +6,7 @@ Five action surfaces:
|
|||||||
DELETE /api/admin/tags/<int:tag_id> (Tier B)
|
DELETE /api/admin/tags/<int:tag_id> (Tier B)
|
||||||
POST /api/admin/tags/<int:dest_id>/merge (Tier B)
|
POST /api/admin/tags/<int:dest_id>/merge (Tier B)
|
||||||
POST /api/admin/tags/prune-unused (Tier A)
|
POST /api/admin/tags/prune-unused (Tier A)
|
||||||
|
POST /api/admin/tags/purge-legacy (Tier A)
|
||||||
GET /api/admin/tags/<int:tag_id>/usage-count (helper)
|
GET /api/admin/tags/<int:tag_id>/usage-count (helper)
|
||||||
|
|
||||||
Tier-C ops take a dry_run body flag (returns projection inline,
|
Tier-C ops take a dry_run body flag (returns projection inline,
|
||||||
@@ -23,16 +24,11 @@ from sqlalchemy import select
|
|||||||
from ..extensions import get_session
|
from ..extensions import get_session
|
||||||
from ..models import Artist
|
from ..models import Artist
|
||||||
from ..services.cleanup_service import project_artist_cascade, project_bulk_image_delete
|
from ..services.cleanup_service import project_artist_cascade, project_bulk_image_delete
|
||||||
|
from ._responses import error_response as _bad
|
||||||
|
|
||||||
admin_bp = Blueprint("admin", __name__, url_prefix="/api/admin")
|
admin_bp = Blueprint("admin", __name__, url_prefix="/api/admin")
|
||||||
|
|
||||||
|
|
||||||
def _bad(error: str, *, status: int = 400, **extra):
|
|
||||||
body = {"error": error}
|
|
||||||
body.update(extra)
|
|
||||||
return jsonify(body), status
|
|
||||||
|
|
||||||
|
|
||||||
def _bulk_image_confirm_token(image_ids: list[int]) -> str:
|
def _bulk_image_confirm_token(image_ids: list[int]) -> str:
|
||||||
"""Stable 8-hex token derived from the sorted id list. Mutates
|
"""Stable 8-hex token derived from the sorted id list. Mutates
|
||||||
when the selection changes; stays the same across modal opens of
|
when the selection changes; stays the same across modal opens of
|
||||||
@@ -206,3 +202,23 @@ async def tags_prune_unused():
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
return jsonify(result)
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
||||||
|
@admin_bp.route("/tags/purge-legacy", methods=["POST"])
|
||||||
|
async def tags_purge_legacy():
|
||||||
|
"""Tier-A: delete legacy IR-migration tags — archive/post/artist
|
||||||
|
kinds (e.g. `BlenderKnight:Hannah_BJ_Loops`) PLUS general tags with
|
||||||
|
a legacy name prefix (`source:*`, from IR's source kind that fell
|
||||||
|
back to general). dry-run preview returns per-kind + per-prefix
|
||||||
|
counts + a sample so the UI shows exactly what'll go before the
|
||||||
|
operator confirms with dry_run=false."""
|
||||||
|
from ..services.cleanup_service import purge_legacy_tags
|
||||||
|
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
dry_run = bool(body.get("dry_run", False))
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
result = await session.run_sync(
|
||||||
|
lambda sync_sess: purge_legacy_tags(sync_sess, dry_run=dry_run)
|
||||||
|
)
|
||||||
|
return jsonify(result)
|
||||||
|
|||||||
@@ -31,18 +31,13 @@ from sqlalchemy import select
|
|||||||
from ..extensions import get_session
|
from ..extensions import get_session
|
||||||
from ..models import LibraryAuditRun
|
from ..models import LibraryAuditRun
|
||||||
from ..services import cleanup_service
|
from ..services import cleanup_service
|
||||||
|
from ._responses import error_response as _bad
|
||||||
|
|
||||||
cleanup_bp = Blueprint("cleanup", __name__, url_prefix="/api/cleanup")
|
cleanup_bp = Blueprint("cleanup", __name__, url_prefix="/api/cleanup")
|
||||||
|
|
||||||
IMAGES_ROOT = Path("/images")
|
IMAGES_ROOT = Path("/images")
|
||||||
|
|
||||||
|
|
||||||
def _bad(error: str, *, status: int = 400, **extra):
|
|
||||||
body = {"error": error}
|
|
||||||
body.update(extra)
|
|
||||||
return jsonify(body), status
|
|
||||||
|
|
||||||
|
|
||||||
def _min_dim_token(min_w: int, min_h: int) -> str:
|
def _min_dim_token(min_w: int, min_h: int) -> str:
|
||||||
# SHA-256 (not MD5) — Web Crypto's subtle.digest rejects MD5; both
|
# SHA-256 (not MD5) — Web Crypto's subtle.digest rejects MD5; both
|
||||||
# sides use SHA-256 truncated to 8 hex chars.
|
# sides use SHA-256 truncated to 8 hex chars.
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ from ..services.credential_service import (
|
|||||||
UnknownPlatformError,
|
UnknownPlatformError,
|
||||||
WrongAuthTypeError,
|
WrongAuthTypeError,
|
||||||
)
|
)
|
||||||
|
from ._responses import error_response as _bad
|
||||||
|
|
||||||
credentials_bp = Blueprint("credentials", __name__, url_prefix="/api/credentials")
|
credentials_bp = Blueprint("credentials", __name__, url_prefix="/api/credentials")
|
||||||
|
|
||||||
@@ -38,14 +39,6 @@ def _get_crypto() -> CredentialCrypto:
|
|||||||
return _crypto
|
return _crypto
|
||||||
|
|
||||||
|
|
||||||
def _bad(error: str, *, status: int = 400, detail: str | None = None, **extra):
|
|
||||||
body = {"error": error}
|
|
||||||
if detail is not None:
|
|
||||||
body["detail"] = detail
|
|
||||||
body.update(extra)
|
|
||||||
return jsonify(body), status
|
|
||||||
|
|
||||||
|
|
||||||
async def _ext_key_ok(session) -> bool:
|
async def _ext_key_ok(session) -> bool:
|
||||||
"""If X-Extension-Key is supplied, it must match the stored value.
|
"""If X-Extension-Key is supplied, it must match the stored value.
|
||||||
Missing header → True (browser path; accepted per homelab posture).
|
Missing header → True (browser path; accepted per homelab posture).
|
||||||
@@ -124,3 +117,56 @@ async def delete_credential(platform: str):
|
|||||||
except LookupError:
|
except LookupError:
|
||||||
return _bad("not_found", status=404)
|
return _bad("not_found", status=404)
|
||||||
return "", 204
|
return "", 204
|
||||||
|
|
||||||
|
|
||||||
|
@credentials_bp.route("/<platform>/verify", methods=["POST"])
|
||||||
|
async def verify_credential(platform: str):
|
||||||
|
"""Test the stored credential by running gallery-dl --simulate
|
||||||
|
against one of the platform's enabled sources. On success stamps
|
||||||
|
last_verified. Returns {valid: bool|null, reason, last_verified?}.
|
||||||
|
valid=null means "couldn't test" (no credential, or no enabled
|
||||||
|
source to point at)."""
|
||||||
|
from ..models import Artist, Source
|
||||||
|
from ..services.gallery_dl import GalleryDLService, SourceConfig
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
if not await _ext_key_ok(session):
|
||||||
|
return _bad("unauthorized", status=401)
|
||||||
|
svc = CredentialService(session, _get_crypto())
|
||||||
|
record = await svc.get(platform)
|
||||||
|
if record is None:
|
||||||
|
return jsonify({"valid": None, "reason": "No credential stored for this platform."})
|
||||||
|
|
||||||
|
# Pick an enabled source for this platform to point the probe at.
|
||||||
|
row = (await session.execute(
|
||||||
|
select(Source, Artist)
|
||||||
|
.join(Artist, Artist.id == Source.artist_id)
|
||||||
|
.where(Source.platform == platform, Source.enabled.is_(True))
|
||||||
|
.order_by(Source.id.asc())
|
||||||
|
)).first()
|
||||||
|
if row is None:
|
||||||
|
return jsonify({
|
||||||
|
"valid": None,
|
||||||
|
"reason": "No enabled source for this platform to verify against — add a subscription first.",
|
||||||
|
})
|
||||||
|
source, artist = row
|
||||||
|
|
||||||
|
cookies_path = await svc.get_cookies_path(platform)
|
||||||
|
auth_token = await svc.get_token(platform)
|
||||||
|
|
||||||
|
gdl = GalleryDLService(images_root=Path("/images"))
|
||||||
|
ok, message = await gdl.verify(
|
||||||
|
url=source.url,
|
||||||
|
artist_slug=artist.slug,
|
||||||
|
platform=platform,
|
||||||
|
source_config=SourceConfig.from_dict(source.config_overrides or {}),
|
||||||
|
cookies_path=str(cookies_path) if cookies_path else None,
|
||||||
|
auth_token=auth_token,
|
||||||
|
)
|
||||||
|
|
||||||
|
last_verified = None
|
||||||
|
if ok:
|
||||||
|
async with get_session() as session:
|
||||||
|
ts = await CredentialService(session, _get_crypto()).mark_verified(platform)
|
||||||
|
last_verified = ts.isoformat() if ts else None
|
||||||
|
return jsonify({"valid": ok, "reason": message, "last_verified": last_verified})
|
||||||
|
|||||||
@@ -126,6 +126,54 @@ async def downloads_stats():
|
|||||||
return jsonify(out)
|
return jsonify(out)
|
||||||
|
|
||||||
|
|
||||||
|
@downloads_bp.route("/activity", methods=["GET"])
|
||||||
|
async def downloads_activity():
|
||||||
|
"""Hourly download-event counts over the last `?hours=` (default 24).
|
||||||
|
|
||||||
|
Returns a fixed-length, oldest-first bucket array so the UI can render
|
||||||
|
a sparkline directly. Bucketing is done in Python against UTC to dodge
|
||||||
|
session-timezone ambiguity in SQL date_trunc.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
hours = int(request.args.get("hours", "24"))
|
||||||
|
except ValueError:
|
||||||
|
return jsonify({"error": "invalid_hours"}), 400
|
||||||
|
hours = max(1, min(168, hours))
|
||||||
|
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
end = now.replace(minute=0, second=0, microsecond=0)
|
||||||
|
start = end - timedelta(hours=hours - 1)
|
||||||
|
buckets = [
|
||||||
|
{"hour": (start + timedelta(hours=i)).isoformat(),
|
||||||
|
"ok": 0, "error": 0, "other": 0, "total": 0}
|
||||||
|
for i in range(hours)
|
||||||
|
]
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
rows = (await session.execute(
|
||||||
|
select(DownloadEvent.started_at, DownloadEvent.status)
|
||||||
|
.where(DownloadEvent.started_at >= start)
|
||||||
|
)).all()
|
||||||
|
|
||||||
|
for started_at, status in rows:
|
||||||
|
if started_at is None:
|
||||||
|
continue
|
||||||
|
sa = started_at if started_at.tzinfo else started_at.replace(tzinfo=UTC)
|
||||||
|
idx = int((sa - start).total_seconds() // 3600)
|
||||||
|
if not (0 <= idx < hours):
|
||||||
|
continue
|
||||||
|
b = buckets[idx]
|
||||||
|
if status == "ok":
|
||||||
|
b["ok"] += 1
|
||||||
|
elif status == "error":
|
||||||
|
b["error"] += 1
|
||||||
|
else:
|
||||||
|
b["other"] += 1
|
||||||
|
b["total"] += 1
|
||||||
|
|
||||||
|
return jsonify({"hours": hours, "buckets": buckets})
|
||||||
|
|
||||||
|
|
||||||
@downloads_bp.route("/<int:event_id>", methods=["GET"])
|
@downloads_bp.route("/<int:event_id>", methods=["GET"])
|
||||||
async def get_download(event_id: int):
|
async def get_download(event_id: int):
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
@@ -139,3 +187,20 @@ async def get_download(event_id: int):
|
|||||||
return jsonify({"error": "not_found"}), 404
|
return jsonify({"error": "not_found"}), 404
|
||||||
event, source, artist = row
|
event, source, artist = row
|
||||||
return jsonify(_detail_record(event, source, artist))
|
return jsonify(_detail_record(event, source, artist))
|
||||||
|
|
||||||
|
|
||||||
|
@downloads_bp.route("/recover-stalled", methods=["POST"])
|
||||||
|
async def recover_stalled():
|
||||||
|
"""Trigger the recover_stalled_download_events sweep on demand.
|
||||||
|
|
||||||
|
The same sweep runs every 5 min via Beat (see celery_app.beat_schedule);
|
||||||
|
this endpoint exists so the operator can force-clear stuck pending/
|
||||||
|
running download_events from the Subscriptions → Downloads maintenance
|
||||||
|
menu without waiting for the next scheduled tick.
|
||||||
|
"""
|
||||||
|
# Local import: avoids registering maintenance tasks during blueprint
|
||||||
|
# import (Celery task discovery races with the API import otherwise).
|
||||||
|
from ..tasks.maintenance import recover_stalled_download_events
|
||||||
|
|
||||||
|
recover_stalled_download_events.delay()
|
||||||
|
return jsonify({"queued": True}), 202
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ from ..services.extension_service import (
|
|||||||
UnknownPlatformError,
|
UnknownPlatformError,
|
||||||
)
|
)
|
||||||
from ..services.source_service import KNOWN_PLATFORMS
|
from ..services.source_service import KNOWN_PLATFORMS
|
||||||
|
from ._responses import error_response as _bad
|
||||||
|
|
||||||
extension_bp = Blueprint("extension", __name__, url_prefix="/api/extension")
|
extension_bp = Blueprint("extension", __name__, url_prefix="/api/extension")
|
||||||
|
|
||||||
@@ -30,12 +31,6 @@ XPI_DIR = Path("/app/frontend/dist/extension")
|
|||||||
_XPI_VERSION_RE = re.compile(r"fabledcurator-(?P<version>[\w.-]+)\.xpi$")
|
_XPI_VERSION_RE = re.compile(r"fabledcurator-(?P<version>[\w.-]+)\.xpi$")
|
||||||
|
|
||||||
|
|
||||||
def _bad(error: str, *, status: int = 400, **extra):
|
|
||||||
body = {"error": error}
|
|
||||||
body.update(extra)
|
|
||||||
return jsonify(body), status
|
|
||||||
|
|
||||||
|
|
||||||
async def _ext_key_required(session) -> bool:
|
async def _ext_key_required(session) -> bool:
|
||||||
"""Unlike /api/credentials (which accepts the browser path with no
|
"""Unlike /api/credentials (which accepts the browser path with no
|
||||||
header), quick-add-source writes server state and must be explicitly
|
header), quick-add-source writes server state and must be explicitly
|
||||||
@@ -62,6 +57,24 @@ def _sha256(path: Path) -> str:
|
|||||||
return h.hexdigest()
|
return h.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
@extension_bp.route("/probe", methods=["GET"])
|
||||||
|
async def probe_source():
|
||||||
|
"""Read-only resolution of a creator-page URL: tells the extension
|
||||||
|
whether this URL is already a Source, is for an Artist that exists
|
||||||
|
but with a different URL, is brand new, or doesn't match any known
|
||||||
|
platform pattern. Drives the content-script chip's color/copy
|
||||||
|
BEFORE the operator clicks, so the button can show 'already added'
|
||||||
|
without requiring an add-attempt."""
|
||||||
|
url = (request.args.get("url") or "").strip()
|
||||||
|
if not url:
|
||||||
|
return _bad("invalid_body", detail="url query parameter is required")
|
||||||
|
async with get_session() as session:
|
||||||
|
if not await _ext_key_required(session):
|
||||||
|
return _bad("unauthorized", status=401)
|
||||||
|
result = await ExtensionService(session).probe(url)
|
||||||
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
||||||
@extension_bp.route("/quick-add-source", methods=["POST"])
|
@extension_bp.route("/quick-add-source", methods=["POST"])
|
||||||
async def quick_add_source():
|
async def quick_add_source():
|
||||||
body = await request.get_json(silent=True)
|
body = await request.get_json(silent=True)
|
||||||
|
|||||||
@@ -114,18 +114,53 @@ async def retry_failed():
|
|||||||
status="queued", error=None,
|
status="queued", error=None,
|
||||||
started_at=None, finished_at=None,
|
started_at=None, finished_at=None,
|
||||||
)
|
)
|
||||||
.returning(ImportTask.id)
|
.returning(ImportTask.id, ImportTask.task_type)
|
||||||
)
|
)
|
||||||
failed_ids = [row[0] for row in result.all()]
|
failed = result.all()
|
||||||
if not failed_ids:
|
if not failed:
|
||||||
return jsonify({"retried": 0})
|
return jsonify({"retried": 0})
|
||||||
await session.commit()
|
await session.commit()
|
||||||
|
|
||||||
from ..tasks.import_file import import_media_file
|
from ..tasks.import_file import enqueue_import
|
||||||
for tid in failed_ids:
|
for tid, task_type in failed:
|
||||||
import_media_file.delay(tid)
|
enqueue_import(tid, task_type)
|
||||||
|
|
||||||
return jsonify({"retried": len(failed_ids)})
|
return jsonify({"retried": len(failed)})
|
||||||
|
|
||||||
|
|
||||||
|
@import_admin_bp.route("/tasks/<int:task_id>/refetch", methods=["POST"])
|
||||||
|
async def refetch_task(task_id: int):
|
||||||
|
"""Layer-2 one-shot re-download: delete the (corrupt) file behind a
|
||||||
|
failed import task and re-run its source's downloader to fetch a
|
||||||
|
fresh copy. Only works for files that resolve to an enabled,
|
||||||
|
real-URL subscription Source; filesystem-only imports return
|
||||||
|
no_source.
|
||||||
|
|
||||||
|
Returns one of: refetch_queued (+source_id) / no_source /
|
||||||
|
already_refetched / not_found / not_failed.
|
||||||
|
"""
|
||||||
|
async with get_session() as session:
|
||||||
|
result = await session.run_sync(_refetch_task_sync, task_id)
|
||||||
|
if result["status"] == "not_found":
|
||||||
|
return jsonify(result), 404
|
||||||
|
if result["status"] == "not_failed":
|
||||||
|
return jsonify(result), 400
|
||||||
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
||||||
|
def _refetch_task_sync(session, task_id: int) -> dict:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ..models import ImportSettings
|
||||||
|
from ..services.refetch_service import attempt_refetch
|
||||||
|
|
||||||
|
task = session.get(ImportTask, task_id)
|
||||||
|
if task is None:
|
||||||
|
return {"status": "not_found"}
|
||||||
|
if task.status != "failed":
|
||||||
|
return {"status": "not_failed"}
|
||||||
|
settings = ImportSettings.load_sync(session)
|
||||||
|
return attempt_refetch(session, task, Path(settings.import_scan_path))
|
||||||
|
|
||||||
|
|
||||||
@import_admin_bp.route("/clear-stuck", methods=["POST"])
|
@import_admin_bp.route("/clear-stuck", methods=["POST"])
|
||||||
|
|||||||
@@ -1,112 +0,0 @@
|
|||||||
"""FC-5: /api/migrate — trigger and poll migration runs.
|
|
||||||
|
|
||||||
Ingest kinds (gs_ingest, ir_ingest) accept multipart/form-data with an
|
|
||||||
`export_file` field. All other kinds accept JSON. Backup + rollback
|
|
||||||
were retired in FC-3h (2026-05-24); use /api/system/backup/* instead.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import json
|
|
||||||
|
|
||||||
from quart import Blueprint, jsonify, request
|
|
||||||
from sqlalchemy import select
|
|
||||||
|
|
||||||
from ..extensions import get_session
|
|
||||||
from ..models import MigrationRun
|
|
||||||
from ..tasks.migration import run_migration
|
|
||||||
|
|
||||||
migrate_bp = Blueprint("migrate", __name__, url_prefix="/api/migrate")
|
|
||||||
|
|
||||||
# 'backup' + 'rollback' retired 2026-05-24 (FC-3h); see /api/system/backup/*.
|
|
||||||
_VALID_KINDS = frozenset({
|
|
||||||
"gs_ingest", "ir_ingest", "tag_apply",
|
|
||||||
"ml_queue", "verify", "cleanup",
|
|
||||||
})
|
|
||||||
_INGEST_KINDS = frozenset({"gs_ingest", "ir_ingest"})
|
|
||||||
|
|
||||||
|
|
||||||
def _bad(error: str, *, status: int = 400, **extra):
|
|
||||||
body = {"error": error}
|
|
||||||
body.update(extra)
|
|
||||||
return jsonify(body), status
|
|
||||||
|
|
||||||
|
|
||||||
def _run_to_dict(run: MigrationRun) -> dict:
|
|
||||||
return {
|
|
||||||
"id": run.id,
|
|
||||||
"kind": run.kind,
|
|
||||||
"status": run.status,
|
|
||||||
"dry_run": run.dry_run,
|
|
||||||
"started_at": run.started_at.isoformat(),
|
|
||||||
"finished_at": run.finished_at.isoformat() if run.finished_at else None,
|
|
||||||
"counts": run.counts or {},
|
|
||||||
"error": run.error,
|
|
||||||
"metadata": run.metadata_ or {},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@migrate_bp.route("/<kind>", methods=["POST"])
|
|
||||||
async def create_run(kind: str):
|
|
||||||
if kind not in _VALID_KINDS:
|
|
||||||
return _bad("unknown_kind", detail=f"kind must be one of {sorted(_VALID_KINDS)}")
|
|
||||||
|
|
||||||
# Ingest kinds accept multipart/form-data; everything else takes JSON.
|
|
||||||
if kind in _INGEST_KINDS:
|
|
||||||
form = await request.form
|
|
||||||
files = await request.files
|
|
||||||
if "export_file" not in files:
|
|
||||||
return _bad("missing_export_file", detail="multipart export_file required")
|
|
||||||
export_file = files["export_file"]
|
|
||||||
try:
|
|
||||||
raw = export_file.read()
|
|
||||||
data = json.loads(raw.decode("utf-8"))
|
|
||||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
||||||
return _bad("invalid_export_file", detail=str(exc))
|
|
||||||
dry_run = str(form.get("dry_run", "false")).lower() in ("true", "1", "yes")
|
|
||||||
params: dict = {"data": data, "dry_run": dry_run}
|
|
||||||
else:
|
|
||||||
body = await request.get_json()
|
|
||||||
if body is None:
|
|
||||||
body = {}
|
|
||||||
if not isinstance(body, dict):
|
|
||||||
return _bad("invalid_body")
|
|
||||||
dry_run = bool(body.get("dry_run", False))
|
|
||||||
params = dict(body)
|
|
||||||
|
|
||||||
async with get_session() as session:
|
|
||||||
run = MigrationRun(kind=kind, status="pending", dry_run=dry_run)
|
|
||||||
session.add(run)
|
|
||||||
await session.commit()
|
|
||||||
await session.refresh(run)
|
|
||||||
run_id = run.id
|
|
||||||
|
|
||||||
run_migration.delay(run_id, kind, params)
|
|
||||||
return jsonify({"run_id": run_id, "status": "pending"}), 202
|
|
||||||
|
|
||||||
|
|
||||||
@migrate_bp.route("/runs/<int:run_id>", methods=["GET"])
|
|
||||||
async def get_run(run_id: int):
|
|
||||||
async with get_session() as session:
|
|
||||||
run = (await session.execute(
|
|
||||||
select(MigrationRun).where(MigrationRun.id == run_id)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if run is None:
|
|
||||||
return _bad("not_found", status=404)
|
|
||||||
return jsonify(_run_to_dict(run))
|
|
||||||
|
|
||||||
|
|
||||||
@migrate_bp.route("/runs", methods=["GET"])
|
|
||||||
async def list_runs():
|
|
||||||
try:
|
|
||||||
limit = int(request.args.get("limit", "10"))
|
|
||||||
except ValueError:
|
|
||||||
return _bad("invalid_limit")
|
|
||||||
if limit < 1 or limit > 100:
|
|
||||||
return _bad("invalid_limit")
|
|
||||||
|
|
||||||
async with get_session() as session:
|
|
||||||
rows = (await session.execute(
|
|
||||||
select(MigrationRun)
|
|
||||||
.order_by(MigrationRun.id.desc())
|
|
||||||
.limit(limit)
|
|
||||||
)).scalars().all()
|
|
||||||
return jsonify([_run_to_dict(r) for r in rows])
|
|
||||||
+25
-11
@@ -5,18 +5,11 @@ from quart import Blueprint, jsonify, request
|
|||||||
from ..extensions import get_session
|
from ..extensions import get_session
|
||||||
from ..services.post_feed_service import PostFeedService
|
from ..services.post_feed_service import PostFeedService
|
||||||
from ..services.source_service import KNOWN_PLATFORMS
|
from ..services.source_service import KNOWN_PLATFORMS
|
||||||
|
from ._responses import error_response as _bad
|
||||||
|
|
||||||
posts_bp = Blueprint("posts", __name__, url_prefix="/api/posts")
|
posts_bp = Blueprint("posts", __name__, url_prefix="/api/posts")
|
||||||
|
|
||||||
|
|
||||||
def _bad(error: str, *, status: int = 400, detail: str | None = None, **extra):
|
|
||||||
body = {"error": error}
|
|
||||||
if detail is not None:
|
|
||||||
body["detail"] = detail
|
|
||||||
body.update(extra)
|
|
||||||
return jsonify(body), status
|
|
||||||
|
|
||||||
|
|
||||||
@posts_bp.route("", methods=["GET"])
|
@posts_bp.route("", methods=["GET"])
|
||||||
async def list_posts():
|
async def list_posts():
|
||||||
args = request.args
|
args = request.args
|
||||||
@@ -25,6 +18,8 @@ async def list_posts():
|
|||||||
artist_id_raw = args.get("artist_id")
|
artist_id_raw = args.get("artist_id")
|
||||||
platform = args.get("platform") or None
|
platform = args.get("platform") or None
|
||||||
limit_raw = args.get("limit", "24")
|
limit_raw = args.get("limit", "24")
|
||||||
|
direction = args.get("direction", "older")
|
||||||
|
around_raw = args.get("around")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
limit = int(limit_raw)
|
limit = int(limit_raw)
|
||||||
@@ -33,6 +28,16 @@ async def list_posts():
|
|||||||
if limit < 1 or limit > 100:
|
if limit < 1 or limit > 100:
|
||||||
return _bad("invalid_limit", detail="limit must be between 1 and 100")
|
return _bad("invalid_limit", detail="limit must be between 1 and 100")
|
||||||
|
|
||||||
|
if direction not in ("older", "newer"):
|
||||||
|
return _bad("invalid_direction", detail="direction must be 'older' or 'newer'")
|
||||||
|
|
||||||
|
around_id = None
|
||||||
|
if around_raw is not None:
|
||||||
|
try:
|
||||||
|
around_id = int(around_raw)
|
||||||
|
except ValueError:
|
||||||
|
return _bad("invalid_around", detail="around must be an integer post id")
|
||||||
|
|
||||||
artist_id = None
|
artist_id = None
|
||||||
if artist_id_raw is not None:
|
if artist_id_raw is not None:
|
||||||
try:
|
try:
|
||||||
@@ -47,11 +52,20 @@ async def list_posts():
|
|||||||
)
|
)
|
||||||
|
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
try:
|
svc = PostFeedService(session)
|
||||||
page = await PostFeedService(session).scroll(
|
if around_id is not None:
|
||||||
cursor=cursor, artist_id=artist_id,
|
result = await svc.around(
|
||||||
|
post_id=around_id, artist_id=artist_id,
|
||||||
platform=platform, limit=limit,
|
platform=platform, limit=limit,
|
||||||
)
|
)
|
||||||
|
if result is None:
|
||||||
|
return _bad("not_found", status=404, detail=f"post id={around_id}")
|
||||||
|
return jsonify(result)
|
||||||
|
try:
|
||||||
|
page = await svc.scroll(
|
||||||
|
cursor=cursor, artist_id=artist_id,
|
||||||
|
platform=platform, limit=limit, direction=direction,
|
||||||
|
)
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
# Service raises ValueError for malformed cursors only;
|
# Service raises ValueError for malformed cursors only;
|
||||||
# limit bounds are validated above.
|
# limit bounds are validated above.
|
||||||
|
|||||||
@@ -31,9 +31,7 @@ _EDITABLE_FIELDS = (
|
|||||||
@settings_bp.route("/settings/import", methods=["GET"])
|
@settings_bp.route("/settings/import", methods=["GET"])
|
||||||
async def get_import_settings():
|
async def get_import_settings():
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
row = (
|
row = await ImportSettings.load(session)
|
||||||
await session.execute(select(ImportSettings).where(ImportSettings.id == 1))
|
|
||||||
).scalar_one()
|
|
||||||
return jsonify({
|
return jsonify({
|
||||||
"min_width": row.min_width,
|
"min_width": row.min_width,
|
||||||
"min_height": row.min_height,
|
"min_height": row.min_height,
|
||||||
@@ -99,9 +97,7 @@ async def update_import_settings():
|
|||||||
return _bad_int("download_failure_warning_threshold", 1, 100)
|
return _bad_int("download_failure_warning_threshold", 1, 100)
|
||||||
|
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
row = (
|
row = await ImportSettings.load(session)
|
||||||
await session.execute(select(ImportSettings).where(ImportSettings.id == 1))
|
|
||||||
).scalar_one()
|
|
||||||
for field in _EDITABLE_FIELDS:
|
for field in _EDITABLE_FIELDS:
|
||||||
if field in body:
|
if field in body:
|
||||||
setattr(row, field, body[field])
|
setattr(row, field, body[field])
|
||||||
|
|||||||
+34
-10
@@ -5,6 +5,7 @@ from sqlalchemy import select
|
|||||||
|
|
||||||
from ..extensions import get_session
|
from ..extensions import get_session
|
||||||
from ..models import DownloadEvent, Source
|
from ..models import DownloadEvent, Source
|
||||||
|
from ..services.scheduler_service import active_platform_cooldowns, scheduler_status
|
||||||
from ..services.source_service import (
|
from ..services.source_service import (
|
||||||
KNOWN_PLATFORMS,
|
KNOWN_PLATFORMS,
|
||||||
ArtistNotFoundError,
|
ArtistNotFoundError,
|
||||||
@@ -14,18 +15,11 @@ from ..services.source_service import (
|
|||||||
SourceService,
|
SourceService,
|
||||||
UnknownPlatformError,
|
UnknownPlatformError,
|
||||||
)
|
)
|
||||||
|
from ._responses import error_response as _bad
|
||||||
|
|
||||||
sources_bp = Blueprint("sources", __name__, url_prefix="/api/sources")
|
sources_bp = Blueprint("sources", __name__, url_prefix="/api/sources")
|
||||||
|
|
||||||
|
|
||||||
def _bad(error: str, *, status: int = 400, detail: str | None = None, **extra):
|
|
||||||
body = {"error": error}
|
|
||||||
if detail is not None:
|
|
||||||
body["detail"] = detail
|
|
||||||
body.update(extra)
|
|
||||||
return jsonify(body), status
|
|
||||||
|
|
||||||
|
|
||||||
@sources_bp.route("", methods=["GET"])
|
@sources_bp.route("", methods=["GET"])
|
||||||
async def list_sources():
|
async def list_sources():
|
||||||
artist_id_raw = request.args.get("artist_id")
|
artist_id_raw = request.args.get("artist_id")
|
||||||
@@ -35,11 +29,19 @@ async def list_sources():
|
|||||||
artist_id = int(artist_id_raw)
|
artist_id = int(artist_id_raw)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
return _bad("invalid_artist_id", detail="artist_id must be an integer")
|
return _bad("invalid_artist_id", detail="artist_id must be an integer")
|
||||||
|
failing = request.args.get("failing", "").lower() in ("1", "true", "yes")
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
records = await SourceService(session).list(artist_id=artist_id)
|
records = await SourceService(session).list(artist_id=artist_id, failing=failing)
|
||||||
return jsonify([r.to_dict() for r in records])
|
return jsonify([r.to_dict() for r in records])
|
||||||
|
|
||||||
|
|
||||||
|
@sources_bp.route("/schedule-status", methods=["GET"])
|
||||||
|
async def schedule_status():
|
||||||
|
"""FC-dashboards: scheduler health for the Subscriptions hub."""
|
||||||
|
async with get_session() as session:
|
||||||
|
return jsonify(await scheduler_status(session))
|
||||||
|
|
||||||
|
|
||||||
@sources_bp.route("/<int:source_id>", methods=["GET"])
|
@sources_bp.route("/<int:source_id>", methods=["GET"])
|
||||||
async def get_source(source_id: int):
|
async def get_source(source_id: int):
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
@@ -123,7 +125,16 @@ async def check_source(source_id: int):
|
|||||||
"""FC-3c: enqueue a download for this source.
|
"""FC-3c: enqueue a download for this source.
|
||||||
|
|
||||||
Returns 202 with the new DownloadEvent id. If a pending/running
|
Returns 202 with the new DownloadEvent id. If a pending/running
|
||||||
event already exists for this source, returns 409 with that id."""
|
event already exists for this source, returns 409 with that id. If
|
||||||
|
the source's platform is currently in a rate-limit cooldown, returns
|
||||||
|
**202 with `{status: "deferred", cooldown_until, platform}`** and
|
||||||
|
does NOT create an event or dispatch — the bulk retry path uses this
|
||||||
|
to avoid bowling N sources right back into the rate limit the
|
||||||
|
cooldown is preventing. Single-click "retry this one source" passes
|
||||||
|
`?force=true` to override the cooldown (operator-explicit, useful
|
||||||
|
for rapid auth-fix testing). The in-flight guard always applies.
|
||||||
|
"""
|
||||||
|
force = (request.args.get("force") or "").lower() in ("1", "true", "yes")
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
source = (await session.execute(
|
source = (await session.execute(
|
||||||
select(Source).where(Source.id == source_id)
|
select(Source).where(Source.id == source_id)
|
||||||
@@ -133,6 +144,19 @@ async def check_source(source_id: int):
|
|||||||
if not source.enabled:
|
if not source.enabled:
|
||||||
return _bad("source_disabled", detail="enable the source first")
|
return _bad("source_disabled", detail="enable the source first")
|
||||||
|
|
||||||
|
# Cooldown gate (unless explicitly overridden). Checked before the
|
||||||
|
# in-flight guard because a deferred retry doesn't need to create
|
||||||
|
# or check for an event at all.
|
||||||
|
if not force:
|
||||||
|
cooldowns = await active_platform_cooldowns(session)
|
||||||
|
expires_at = cooldowns.get(source.platform)
|
||||||
|
if expires_at is not None:
|
||||||
|
return jsonify({
|
||||||
|
"status": "deferred",
|
||||||
|
"platform": source.platform,
|
||||||
|
"cooldown_until": expires_at.isoformat(),
|
||||||
|
}), 202
|
||||||
|
|
||||||
in_flight = (await session.execute(
|
in_flight = (await session.execute(
|
||||||
select(DownloadEvent.id).where(
|
select(DownloadEvent.id).where(
|
||||||
DownloadEvent.source_id == source_id,
|
DownloadEvent.source_id == source_id,
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ from sqlalchemy import desc, func, select
|
|||||||
from ..config import get_config
|
from ..config import get_config
|
||||||
from ..extensions import get_session
|
from ..extensions import get_session
|
||||||
from ..models import TaskRun
|
from ..models import TaskRun
|
||||||
|
from ..services.scheduler_service import scheduler_status
|
||||||
|
|
||||||
system_activity_bp = Blueprint(
|
system_activity_bp = Blueprint(
|
||||||
"system_activity", __name__, url_prefix="/api/system/activity",
|
"system_activity", __name__, url_prefix="/api/system/activity",
|
||||||
@@ -81,17 +82,22 @@ def _read_workers_sync() -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _queues_cached() -> dict:
|
||||||
|
"""Per-queue Redis LLEN, cached 2s. Shared by /queues and /summary."""
|
||||||
|
now = time.time()
|
||||||
|
if _QUEUE_CACHE["data"] is None or (now - _QUEUE_CACHE["ts"]) > _QUEUE_CACHE_TTL:
|
||||||
|
_QUEUE_CACHE["data"] = await asyncio.to_thread(_read_queues_sync)
|
||||||
|
_QUEUE_CACHE["ts"] = now
|
||||||
|
return _QUEUE_CACHE["data"]
|
||||||
|
|
||||||
|
|
||||||
@system_activity_bp.route("/queues", methods=["GET"])
|
@system_activity_bp.route("/queues", methods=["GET"])
|
||||||
async def get_queues():
|
async def get_queues():
|
||||||
"""Per-queue Redis LLEN. Cached 2s.
|
"""Per-queue Redis LLEN. Cached 2s.
|
||||||
|
|
||||||
Response: {queues: {name: depth_or_null}, fetched_at: iso8601}
|
Response: {queues: {name: depth_or_null}, fetched_at: iso8601}
|
||||||
"""
|
"""
|
||||||
now = time.time()
|
return jsonify(await _queues_cached())
|
||||||
if _QUEUE_CACHE["data"] is None or (now - _QUEUE_CACHE["ts"]) > _QUEUE_CACHE_TTL:
|
|
||||||
_QUEUE_CACHE["data"] = await asyncio.to_thread(_read_queues_sync)
|
|
||||||
_QUEUE_CACHE["ts"] = now
|
|
||||||
return jsonify(_QUEUE_CACHE["data"])
|
|
||||||
|
|
||||||
|
|
||||||
@system_activity_bp.route("/workers", methods=["GET"])
|
@system_activity_bp.route("/workers", methods=["GET"])
|
||||||
@@ -107,6 +113,35 @@ async def get_workers():
|
|||||||
return jsonify(_WORKER_CACHE["data"])
|
return jsonify(_WORKER_CACHE["data"])
|
||||||
|
|
||||||
|
|
||||||
|
@system_activity_bp.route("/summary", methods=["GET"])
|
||||||
|
async def get_summary():
|
||||||
|
"""One-call rollup for the always-on TopNav pipeline indicator:
|
||||||
|
scheduler health, per-queue pending depths, currently-running count, and
|
||||||
|
recent (24h) failure count. Cheap — cached queue LLENs + two TaskRun
|
||||||
|
counts — so it's safe to poll app-wide."""
|
||||||
|
queues_data = await _queues_cached()
|
||||||
|
depths = queues_data.get("queues", {})
|
||||||
|
queued_total = sum(v for v in depths.values() if isinstance(v, int))
|
||||||
|
since = datetime.now(UTC) - timedelta(hours=24)
|
||||||
|
async with get_session() as session:
|
||||||
|
scheduler = await scheduler_status(session)
|
||||||
|
running = (await session.execute(
|
||||||
|
select(func.count(TaskRun.id)).where(TaskRun.status == "running")
|
||||||
|
)).scalar_one()
|
||||||
|
failing = (await session.execute(
|
||||||
|
select(func.count(TaskRun.id))
|
||||||
|
.where(TaskRun.status.in_(["error", "timeout"]))
|
||||||
|
.where(TaskRun.finished_at >= since)
|
||||||
|
)).scalar_one()
|
||||||
|
return jsonify({
|
||||||
|
"scheduler": scheduler,
|
||||||
|
"queues": depths,
|
||||||
|
"queued_total": queued_total,
|
||||||
|
"running": int(running),
|
||||||
|
"failing": int(failing),
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
@system_activity_bp.route("/runs", methods=["GET"])
|
@system_activity_bp.route("/runs", methods=["GET"])
|
||||||
async def list_runs():
|
async def list_runs():
|
||||||
"""Paginated task_run history. Query params:
|
"""Paginated task_run history. Query params:
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from sqlalchemy import desc, select
|
|||||||
|
|
||||||
from ..extensions import get_session
|
from ..extensions import get_session
|
||||||
from ..models import BackupRun, ImportSettings
|
from ..models import BackupRun, ImportSettings
|
||||||
|
from ._responses import error_response as _bad
|
||||||
|
|
||||||
system_backup_bp = Blueprint(
|
system_backup_bp = Blueprint(
|
||||||
"system_backup", __name__, url_prefix="/api/system/backup",
|
"system_backup", __name__, url_prefix="/api/system/backup",
|
||||||
@@ -29,12 +30,6 @@ _BACKUP_SETTINGS_FIELDS = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _bad(error: str, *, status: int = 400, **extra):
|
|
||||||
body = {"error": error}
|
|
||||||
body.update(extra)
|
|
||||||
return jsonify(body), status
|
|
||||||
|
|
||||||
|
|
||||||
def _row_to_dict(r: BackupRun) -> dict:
|
def _row_to_dict(r: BackupRun) -> dict:
|
||||||
return {
|
return {
|
||||||
"id": r.id,
|
"id": r.id,
|
||||||
@@ -232,9 +227,7 @@ async def delete_run(run_id: int):
|
|||||||
@system_backup_bp.route("/settings", methods=["GET"])
|
@system_backup_bp.route("/settings", methods=["GET"])
|
||||||
async def get_settings():
|
async def get_settings():
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
row = (await session.execute(
|
row = await ImportSettings.load(session)
|
||||||
select(ImportSettings).where(ImportSettings.id == 1)
|
|
||||||
)).scalar_one()
|
|
||||||
return jsonify({
|
return jsonify({
|
||||||
"backup_db_nightly_enabled": row.backup_db_nightly_enabled,
|
"backup_db_nightly_enabled": row.backup_db_nightly_enabled,
|
||||||
"backup_db_nightly_hour_utc": row.backup_db_nightly_hour_utc,
|
"backup_db_nightly_hour_utc": row.backup_db_nightly_hour_utc,
|
||||||
@@ -254,9 +247,7 @@ async def patch_settings():
|
|||||||
return err
|
return err
|
||||||
|
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
row = (await session.execute(
|
row = await ImportSettings.load(session)
|
||||||
select(ImportSettings).where(ImportSettings.id == 1)
|
|
||||||
)).scalar_one()
|
|
||||||
for field in _BACKUP_SETTINGS_FIELDS:
|
for field in _BACKUP_SETTINGS_FIELDS:
|
||||||
if field in body:
|
if field in body:
|
||||||
setattr(row, field, body[field])
|
setattr(row, field, body[field])
|
||||||
|
|||||||
@@ -28,7 +28,6 @@ def make_celery() -> Celery:
|
|||||||
"backend.app.tasks.import_file",
|
"backend.app.tasks.import_file",
|
||||||
"backend.app.tasks.thumbnail",
|
"backend.app.tasks.thumbnail",
|
||||||
"backend.app.tasks.maintenance",
|
"backend.app.tasks.maintenance",
|
||||||
"backend.app.tasks.migration",
|
|
||||||
"backend.app.tasks.ml",
|
"backend.app.tasks.ml",
|
||||||
"backend.app.tasks.download",
|
"backend.app.tasks.download",
|
||||||
"backend.app.tasks.backup",
|
"backend.app.tasks.backup",
|
||||||
@@ -45,7 +44,6 @@ def make_celery() -> Celery:
|
|||||||
"backend.app.tasks.download.*": {"queue": "download"},
|
"backend.app.tasks.download.*": {"queue": "download"},
|
||||||
"backend.app.tasks.scan.*": {"queue": "scan"},
|
"backend.app.tasks.scan.*": {"queue": "scan"},
|
||||||
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
||||||
"backend.app.tasks.migration.*": {"queue": "maintenance"},
|
|
||||||
"backend.app.tasks.backup.*": {"queue": "maintenance"},
|
"backend.app.tasks.backup.*": {"queue": "maintenance"},
|
||||||
"backend.app.tasks.admin.*": {"queue": "maintenance"},
|
"backend.app.tasks.admin.*": {"queue": "maintenance"},
|
||||||
"backend.app.tasks.library_audit.*": {"queue": "maintenance"},
|
"backend.app.tasks.library_audit.*": {"queue": "maintenance"},
|
||||||
@@ -87,6 +85,10 @@ def make_celery() -> Celery:
|
|||||||
"task": "backend.app.tasks.maintenance.cleanup_old_download_events",
|
"task": "backend.app.tasks.maintenance.cleanup_old_download_events",
|
||||||
"schedule": 86400.0, # daily
|
"schedule": 86400.0, # daily
|
||||||
},
|
},
|
||||||
|
"recover-stalled-download-events": {
|
||||||
|
"task": "backend.app.tasks.maintenance.recover_stalled_download_events",
|
||||||
|
"schedule": 300.0, # every 5 min, matches recover-interrupted-tasks
|
||||||
|
},
|
||||||
"recover-stalled-task-runs": {
|
"recover-stalled-task-runs": {
|
||||||
"task": "backend.app.tasks.maintenance.recover_stalled_task_runs",
|
"task": "backend.app.tasks.maintenance.recover_stalled_task_runs",
|
||||||
"schedule": 300.0, # every 5 min, matches recover-interrupted-tasks
|
"schedule": 300.0, # every 5 min, matches recover-interrupted-tasks
|
||||||
|
|||||||
@@ -66,10 +66,7 @@ def _queue_for(task) -> str:
|
|||||||
return "download"
|
return "download"
|
||||||
if name.startswith("backend.app.tasks.scan."):
|
if name.startswith("backend.app.tasks.scan."):
|
||||||
return "scan"
|
return "scan"
|
||||||
if name.startswith((
|
if name.startswith("backend.app.tasks.maintenance."):
|
||||||
"backend.app.tasks.maintenance.",
|
|
||||||
"backend.app.tasks.migration.",
|
|
||||||
)):
|
|
||||||
return "maintenance"
|
return "maintenance"
|
||||||
return "default"
|
return "default"
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ from .import_batch import ImportBatch
|
|||||||
from .import_settings import ImportSettings
|
from .import_settings import ImportSettings
|
||||||
from .import_task import ImportTask
|
from .import_task import ImportTask
|
||||||
from .library_audit_run import LibraryAuditRun
|
from .library_audit_run import LibraryAuditRun
|
||||||
from .migration_run import MigrationRun
|
|
||||||
from .ml_settings import MLSettings
|
from .ml_settings import MLSettings
|
||||||
from .post import Post
|
from .post import Post
|
||||||
from .post_attachment import PostAttachment
|
from .post_attachment import PostAttachment
|
||||||
@@ -46,7 +45,6 @@ __all__ = [
|
|||||||
"ImportSettings",
|
"ImportSettings",
|
||||||
"LibraryAuditRun",
|
"LibraryAuditRun",
|
||||||
"MLSettings",
|
"MLSettings",
|
||||||
"MigrationRun",
|
|
||||||
"TagAlias",
|
"TagAlias",
|
||||||
"TagAllowlist",
|
"TagAllowlist",
|
||||||
"TagReferenceEmbedding",
|
"TagReferenceEmbedding",
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ Enforced as a single row via a CHECK (id = 1) constraint. The application
|
|||||||
always SELECTs id=1 and never inserts/deletes after the initial migration.
|
always SELECTs id=1 and never inserts/deletes after the initial migration.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from sqlalchemy import Boolean, CheckConstraint, Float, Integer, Text
|
from sqlalchemy import Boolean, CheckConstraint, Float, Integer, Text, select
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
@@ -63,3 +63,13 @@ class ImportSettings(Base):
|
|||||||
backup_images_keep_last_n: Mapped[int] = mapped_column(
|
backup_images_keep_last_n: Mapped[int] = mapped_column(
|
||||||
Integer, nullable=False, default=3,
|
Integer, nullable=False, default=3,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
async def load(cls, session) -> ImportSettings:
|
||||||
|
"""The singleton settings row (id=1), via an async session."""
|
||||||
|
return (await session.execute(select(cls).where(cls.id == 1))).scalar_one()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def load_sync(cls, session) -> ImportSettings:
|
||||||
|
"""The singleton settings row (id=1), via a sync session."""
|
||||||
|
return session.execute(select(cls).where(cls.id == 1)).scalar_one()
|
||||||
|
|||||||
@@ -8,7 +8,16 @@ been processing longer than the stuck-task threshold.
|
|||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, String, Text, func
|
from sqlalchemy import (
|
||||||
|
BigInteger,
|
||||||
|
Boolean,
|
||||||
|
DateTime,
|
||||||
|
ForeignKey,
|
||||||
|
Integer,
|
||||||
|
String,
|
||||||
|
Text,
|
||||||
|
func,
|
||||||
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
@@ -26,6 +35,13 @@ class ImportTask(Base):
|
|||||||
task_type: Mapped[str] = mapped_column(String(16), nullable=False) # media|archive
|
task_type: Mapped[str] = mapped_column(String(16), nullable=False) # media|archive
|
||||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True)
|
status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True)
|
||||||
|
|
||||||
|
# Poison-pill circuit breaker (alembic 0026). recovery_count tracks
|
||||||
|
# how many times the stuck-task sweep has re-queued this row; after
|
||||||
|
# the cap it's failed with a diagnostic instead of looping. refetched
|
||||||
|
# bounds the one-shot re-download remediation to a single attempt.
|
||||||
|
recovery_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
refetched: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||||
|
|
||||||
result_image_id: Mapped[int | None] = mapped_column(
|
result_image_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("image_record.id", ondelete="SET NULL"), nullable=True
|
ForeignKey("image_record.id", ondelete="SET NULL"), nullable=True
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,37 +0,0 @@
|
|||||||
"""MigrationRun — tracks each FC-5 migration invocation (backup/gs/ir/etc).
|
|
||||||
|
|
||||||
kind/status are String(32) not Postgres ENUM so adding kinds later
|
|
||||||
doesn't need a schema migration. The API layer validates values.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from sqlalchemy import Boolean, DateTime, Integer, String, Text, func
|
|
||||||
from sqlalchemy.dialects.postgresql import JSONB
|
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
|
||||||
|
|
||||||
from .base import Base
|
|
||||||
|
|
||||||
|
|
||||||
class MigrationRun(Base):
|
|
||||||
__tablename__ = "migration_run"
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
|
||||||
kind: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
|
||||||
status: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
|
||||||
dry_run: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
||||||
started_at: Mapped[datetime] = mapped_column(
|
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now(),
|
|
||||||
)
|
|
||||||
finished_at: Mapped[datetime | None] = mapped_column(
|
|
||||||
DateTime(timezone=True), nullable=True,
|
|
||||||
)
|
|
||||||
counts: Mapped[dict] = mapped_column(
|
|
||||||
JSONB, nullable=False, default=dict, server_default=sa.text("'{}'::jsonb"),
|
|
||||||
)
|
|
||||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
||||||
metadata_: Mapped[dict] = mapped_column(
|
|
||||||
"metadata", JSONB, nullable=False, default=dict,
|
|
||||||
server_default=sa.text("'{}'::jsonb"),
|
|
||||||
)
|
|
||||||
@@ -127,17 +127,20 @@ class ArtistDirectoryService:
|
|||||||
ImageRecord.artist_id.label("artist_id"),
|
ImageRecord.artist_id.label("artist_id"),
|
||||||
ImageRecord.sha256.label("sha256"),
|
ImageRecord.sha256.label("sha256"),
|
||||||
ImageRecord.mime.label("mime"),
|
ImageRecord.mime.label("mime"),
|
||||||
|
ImageRecord.thumbnail_path.label("thumbnail_path"),
|
||||||
rn,
|
rn,
|
||||||
)
|
)
|
||||||
.where(ImageRecord.artist_id.in_(artist_ids))
|
.where(ImageRecord.artist_id.in_(artist_ids))
|
||||||
.subquery()
|
.subquery()
|
||||||
)
|
)
|
||||||
stmt = (
|
stmt = (
|
||||||
select(sub.c.artist_id, sub.c.sha256, sub.c.mime)
|
select(
|
||||||
|
sub.c.artist_id, sub.c.sha256, sub.c.mime, sub.c.thumbnail_path,
|
||||||
|
)
|
||||||
.where(sub.c.rn <= _PREVIEW_COUNT)
|
.where(sub.c.rn <= _PREVIEW_COUNT)
|
||||||
.order_by(sub.c.artist_id, sub.c.rn)
|
.order_by(sub.c.artist_id, sub.c.rn)
|
||||||
)
|
)
|
||||||
out: dict[int, list[str]] = {}
|
out: dict[int, list[str]] = {}
|
||||||
for aid, sha, mime in (await self.session.execute(stmt)).all():
|
for aid, sha, mime, tp in (await self.session.execute(stmt)).all():
|
||||||
out.setdefault(aid, []).append(thumbnail_url(sha, mime))
|
out.setdefault(aid, []).append(thumbnail_url(tp, sha, mime))
|
||||||
return out
|
return out
|
||||||
|
|||||||
@@ -198,7 +198,7 @@ class ArtistService:
|
|||||||
"mime": r.mime,
|
"mime": r.mime,
|
||||||
"width": r.width,
|
"width": r.width,
|
||||||
"height": r.height,
|
"height": r.height,
|
||||||
"thumbnail_url": thumbnail_url(r.sha256, r.mime),
|
"thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime),
|
||||||
}
|
}
|
||||||
for r in rows
|
for r in rows
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -6,9 +6,8 @@ HTTP handlers (small ops) and from Celery tasks in
|
|||||||
backend.app.tasks.admin (long ops).
|
backend.app.tasks.admin (long ops).
|
||||||
|
|
||||||
This module is the PERMANENT home of artist-cascade + image-unlink
|
This module is the PERMANENT home of artist-cascade + image-unlink
|
||||||
logic. The legacy copy at backend/app/services/migrators/cleanup.py
|
logic. (The legacy migrators/cleanup.py copy was removed with the rest of
|
||||||
stays in place until FC-3j; FC-3j will replace its body with thin
|
the one-and-done GS/IR migration tooling.)
|
||||||
re-exports from this module and then delete the wrapper.
|
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -16,7 +15,7 @@ from datetime import UTC, datetime
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy import func, select, update
|
from sqlalchemy import func, or_, select, update
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from ..models import Artist, ImageRecord, LibraryAuditRun, Tag
|
from ..models import Artist, ImageRecord, LibraryAuditRun, Tag
|
||||||
@@ -369,6 +368,72 @@ def prune_unused_tags(session: Session, *, dry_run: bool = False) -> dict:
|
|||||||
return {"deleted": len(ids), "sample_names": sample}
|
return {"deleted": len(ids), "sample_names": sample}
|
||||||
|
|
||||||
|
|
||||||
|
# Legacy tags FC no longer uses, in two shapes:
|
||||||
|
# (1) kinds the tag input never produces — archive/post/artist.
|
||||||
|
# provenance (post grouping) + archive membership are their own
|
||||||
|
# systems now, and artists are first-class Artist/Source rows.
|
||||||
|
# meta/rating were already hard-deleted by alembic 0023.
|
||||||
|
# (2) name prefixes from IR kinds FC never adopted — `source:*`.
|
||||||
|
# ImageRepo had a `source` kind; FC's enum doesn't, so ir_ingest
|
||||||
|
# fell those back to `general` (kind=general, name="source:patreon"
|
||||||
|
# etc.). They can't be caught by kind, so we match the name prefix.
|
||||||
|
PURGEABLE_TAG_KINDS = ("archive", "post", "artist")
|
||||||
|
LEGACY_NAME_PREFIXES = ("source:",)
|
||||||
|
|
||||||
|
|
||||||
|
def _legacy_tag_predicate():
|
||||||
|
name_clauses = [Tag.name.like(f"{p}%") for p in LEGACY_NAME_PREFIXES]
|
||||||
|
return or_(Tag.kind.in_(PURGEABLE_TAG_KINDS), *name_clauses)
|
||||||
|
|
||||||
|
|
||||||
|
def purge_legacy_tags(session: Session, *, dry_run: bool = False) -> dict:
|
||||||
|
"""Count (dry_run) or delete legacy IR-migration tags: archive/post/
|
||||||
|
artist-kind tags PLUS general tags whose name matches a legacy
|
||||||
|
prefix (source:*).
|
||||||
|
|
||||||
|
CASCADE on image_tag / tag_alias / tag_allowlist /
|
||||||
|
tag_reference_embedding / tag_suggestion_rejection / series_page
|
||||||
|
clears the related rows on the parent DELETE.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
{"by_kind": {kind: count, ...}, # kind-matched rows
|
||||||
|
"by_prefix": {"source:*": count}, # name-prefix-matched rows
|
||||||
|
"count": total, "sample_names": [first 50],
|
||||||
|
and on live runs "deleted": total}
|
||||||
|
"""
|
||||||
|
predicate = _legacy_tag_predicate()
|
||||||
|
rows = session.execute(
|
||||||
|
select(Tag.id, Tag.name, Tag.kind).where(predicate)
|
||||||
|
).all()
|
||||||
|
by_kind: dict[str, int] = {}
|
||||||
|
by_prefix: dict[str, int] = {}
|
||||||
|
for _id, name, kind in rows:
|
||||||
|
# Classify by name-prefix first so a source:* row counts once,
|
||||||
|
# under the prefix bucket, regardless of its (general) kind.
|
||||||
|
matched_prefix = next(
|
||||||
|
(p for p in LEGACY_NAME_PREFIXES if name.startswith(p)), None,
|
||||||
|
)
|
||||||
|
if matched_prefix is not None:
|
||||||
|
label = f"{matched_prefix}*"
|
||||||
|
by_prefix[label] = by_prefix.get(label, 0) + 1
|
||||||
|
else:
|
||||||
|
key = kind.value if hasattr(kind, "value") else str(kind)
|
||||||
|
by_kind[key] = by_kind.get(key, 0) + 1
|
||||||
|
sample = [name for _id, name, _kind in rows[:50]]
|
||||||
|
total = len(rows)
|
||||||
|
result = {
|
||||||
|
"by_kind": by_kind, "by_prefix": by_prefix,
|
||||||
|
"count": total, "sample_names": sample,
|
||||||
|
}
|
||||||
|
if dry_run:
|
||||||
|
return result
|
||||||
|
if total:
|
||||||
|
session.execute(Tag.__table__.delete().where(predicate))
|
||||||
|
session.commit()
|
||||||
|
result["deleted"] = total
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# FC-Cleanup additions (2026-05-26): retroactive audit of import-filter rules.
|
# FC-Cleanup additions (2026-05-26): retroactive audit of import-filter rules.
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
@@ -163,6 +163,19 @@ class CredentialService:
|
|||||||
return None
|
return None
|
||||||
return self.crypto.decrypt(row.encrypted_blob)
|
return self.crypto.decrypt(row.encrypted_blob)
|
||||||
|
|
||||||
|
async def mark_verified(self, platform: str) -> datetime | None:
|
||||||
|
"""Stamp last_verified=now after a successful verify. Returns the
|
||||||
|
timestamp, or None if the credential is gone."""
|
||||||
|
row = (await self.session.execute(
|
||||||
|
select(Credential).where(Credential.platform == platform)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if row is None:
|
||||||
|
return None
|
||||||
|
ts = datetime.now(UTC)
|
||||||
|
row.last_verified = ts
|
||||||
|
await self.session.commit()
|
||||||
|
return ts
|
||||||
|
|
||||||
|
|
||||||
def _augment_cookies(platform: str, netscape: str) -> str:
|
def _augment_cookies(platform: str, netscape: str) -> str:
|
||||||
"""Delegate to the platform's `augment_cookies` hook if one is
|
"""Delegate to the platform's `augment_cookies` hook if one is
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ from .credential_service import CredentialService
|
|||||||
from .gallery_dl import GalleryDLService, SourceConfig
|
from .gallery_dl import GalleryDLService, SourceConfig
|
||||||
from .importer import Importer
|
from .importer import Importer
|
||||||
from .patreon_resolver import resolve_campaign_id
|
from .patreon_resolver import resolve_campaign_id
|
||||||
|
from .scheduler_service import set_platform_cooldown
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -276,18 +277,27 @@ class DownloadService:
|
|||||||
}
|
}
|
||||||
await self._update_source_health(
|
await self._update_source_health(
|
||||||
source_id=ctx["source_id"], status=status, error_message=ev.error,
|
source_id=ctx["source_id"], status=status, error_message=ev.error,
|
||||||
|
error_type=dl_result.error_type.value if dl_result.error_type else None,
|
||||||
)
|
)
|
||||||
await self.async_session.commit()
|
await self.async_session.commit()
|
||||||
return event_id
|
return event_id
|
||||||
|
|
||||||
async def _update_source_health(
|
async def _update_source_health(
|
||||||
self, *, source_id: int, status: str, error_message: str | None,
|
self, *, source_id: int, status: str, error_message: str | None,
|
||||||
|
error_type: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""FC-3d: update Source.{consecutive_failures, last_error, last_checked_at}.
|
"""FC-3d: update Source.{consecutive_failures, last_error, last_checked_at}.
|
||||||
|
|
||||||
ok -> failures = 0, error = None, checked_at = now
|
ok -> failures = 0, error = None, checked_at = now
|
||||||
error -> failures += 1, error = error_message, checked_at = now
|
error -> failures += 1, error = error_message, checked_at = now
|
||||||
skipped -> failures unchanged, error = None, checked_at = now
|
skipped -> failures unchanged, error = None, checked_at = now
|
||||||
|
|
||||||
|
When error_type == 'rate_limited', also stamps a platform-wide
|
||||||
|
cooldown via scheduler_service.set_platform_cooldown so the next
|
||||||
|
scan tick skips every source on this platform until the cooldown
|
||||||
|
expires. Preventive half of the burst-prevention pair —
|
||||||
|
consecutive_failures still backs the offending source off across
|
||||||
|
ticks.
|
||||||
"""
|
"""
|
||||||
source = (await self.async_session.execute(
|
source = (await self.async_session.execute(
|
||||||
select(Source).where(Source.id == source_id)
|
select(Source).where(Source.id == source_id)
|
||||||
@@ -299,6 +309,8 @@ class DownloadService:
|
|||||||
elif status == "error":
|
elif status == "error":
|
||||||
source.consecutive_failures = (source.consecutive_failures or 0) + 1
|
source.consecutive_failures = (source.consecutive_failures or 0) + 1
|
||||||
source.last_error = error_message
|
source.last_error = error_message
|
||||||
|
if error_type == "rate_limited":
|
||||||
|
await set_platform_cooldown(self.async_session, source.platform)
|
||||||
elif status == "skipped":
|
elif status == "skipped":
|
||||||
source.last_error = None
|
source.last_error = None
|
||||||
source.last_checked_at = now
|
source.last_checked_at = now
|
||||||
|
|||||||
@@ -86,6 +86,67 @@ class ExtensionService:
|
|||||||
"created_artist": created_artist,
|
"created_artist": created_artist,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async def probe(self, url: str) -> dict:
|
||||||
|
"""Read-only resolution of a creator-page URL against the FC DB.
|
||||||
|
Returns one of:
|
||||||
|
- {state: 'unknown_platform'} — URL didn't match any
|
||||||
|
platform's strict artist-page pattern
|
||||||
|
- {state: 'new', platform, slug} — would create both
|
||||||
|
artist and source on quick-add
|
||||||
|
- {state: 'artist_match', platform, slug, artist}
|
||||||
|
— artist exists, this
|
||||||
|
exact URL isn't a Source yet (collapses the sidecar-synthetic
|
||||||
|
case too — the synthetic anchor counts as an existing artist
|
||||||
|
row but not as a pollable Source for this URL)
|
||||||
|
- {state: 'source_match', platform, slug, artist, source}
|
||||||
|
— exact (artist, platform,
|
||||||
|
url) Source already exists
|
||||||
|
|
||||||
|
Side-effect-free: two SELECTs at most.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
platform, raw_slug = self._derive(url)
|
||||||
|
except (UnknownPlatformError, InvalidUrlError):
|
||||||
|
return {"state": "unknown_platform"}
|
||||||
|
|
||||||
|
slug = slugify(raw_slug)
|
||||||
|
artist = (await self.session.execute(
|
||||||
|
select(Artist).where(Artist.slug == slug)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if artist is None:
|
||||||
|
return {"state": "new", "platform": platform, "slug": slug}
|
||||||
|
|
||||||
|
artist_payload = {"id": artist.id, "name": artist.name, "slug": artist.slug}
|
||||||
|
|
||||||
|
source = (await self.session.execute(
|
||||||
|
select(Source).where(
|
||||||
|
Source.artist_id == artist.id,
|
||||||
|
Source.platform == platform,
|
||||||
|
Source.url == url,
|
||||||
|
)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if source is None:
|
||||||
|
return {
|
||||||
|
"state": "artist_match",
|
||||||
|
"platform": platform,
|
||||||
|
"slug": slug,
|
||||||
|
"artist": artist_payload,
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
"state": "source_match",
|
||||||
|
"platform": platform,
|
||||||
|
"slug": slug,
|
||||||
|
"artist": artist_payload,
|
||||||
|
"source": {
|
||||||
|
"id": source.id,
|
||||||
|
"artist_id": source.artist_id,
|
||||||
|
"platform": source.platform,
|
||||||
|
"url": source.url,
|
||||||
|
"enabled": source.enabled,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
def _derive(self, url: str) -> tuple[str, str]:
|
def _derive(self, url: str) -> tuple[str, str]:
|
||||||
if not isinstance(url, str) or not url.strip():
|
if not isinstance(url, str) or not url.strip():
|
||||||
raise InvalidUrlError("url is empty")
|
raise InvalidUrlError("url is empty")
|
||||||
|
|||||||
@@ -42,6 +42,18 @@ class ErrorType(StrEnum):
|
|||||||
UNKNOWN_ERROR = "unknown_error"
|
UNKNOWN_ERROR = "unknown_error"
|
||||||
|
|
||||||
|
|
||||||
|
# 30 seconds shy of download_source's Celery soft_time_limit (900s, see
|
||||||
|
# tasks/download.py:32). subprocess.run MUST raise TimeoutExpired before
|
||||||
|
# Celery raises SoftTimeLimitExceeded — otherwise Celery wins the race,
|
||||||
|
# SIGKILLs the worker, in-memory stdout/stderr is lost, and the
|
||||||
|
# DownloadEvent ends up empty-logged with "stranded by recovery sweep"
|
||||||
|
# 18 minutes later (operator-flagged 2026-05-31, Knuxy event #38275).
|
||||||
|
# The 30s buffer absorbs scheduler jitter / GC pauses without making
|
||||||
|
# legitimately-long-running syncs timeout-friendlier. Per-source bumps
|
||||||
|
# still live in source.config_overrides for legitimately long syncs.
|
||||||
|
_DEFAULT_GDL_TIMEOUT_SECONDS = 870
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class SourceConfig:
|
class SourceConfig:
|
||||||
content_types: list[str] = field(default_factory=lambda: ["all"])
|
content_types: list[str] = field(default_factory=lambda: ["all"])
|
||||||
@@ -51,7 +63,7 @@ class SourceConfig:
|
|||||||
filename_pattern: str | None = None
|
filename_pattern: str | None = None
|
||||||
skip_existing: bool = True
|
skip_existing: bool = True
|
||||||
save_metadata: bool = True
|
save_metadata: bool = True
|
||||||
timeout: int = 3600
|
timeout: int = _DEFAULT_GDL_TIMEOUT_SECONDS
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_dict(cls, data: dict) -> SourceConfig:
|
def from_dict(cls, data: dict) -> SourceConfig:
|
||||||
@@ -63,7 +75,7 @@ class SourceConfig:
|
|||||||
filename_pattern=data.get("filename_pattern"),
|
filename_pattern=data.get("filename_pattern"),
|
||||||
skip_existing=data.get("skip_existing", True),
|
skip_existing=data.get("skip_existing", True),
|
||||||
save_metadata=data.get("save_metadata", True),
|
save_metadata=data.get("save_metadata", True),
|
||||||
timeout=data.get("timeout", 3600),
|
timeout=data.get("timeout", _DEFAULT_GDL_TIMEOUT_SECONDS),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -360,7 +372,17 @@ class GalleryDLService:
|
|||||||
if return_code in (1, 4) and (skip_line_count > 0 or has_skip_text) and not has_actual_error:
|
if return_code in (1, 4) and (skip_line_count > 0 or has_skip_text) and not has_actual_error:
|
||||||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||||||
|
|
||||||
if return_code in (1, 4) and not has_actual_error:
|
# Tier-gated classification used to require `return_code in (1, 4)`,
|
||||||
|
# which silently fell through to UNKNOWN_ERROR when gallery-dl
|
||||||
|
# returned a different exit code for mixed-failure runs (e.g.
|
||||||
|
# paywall warnings + a missing yt-dlp dep flipping the exit bits).
|
||||||
|
# The artist then surfaced as "needs attention" purely because a
|
||||||
|
# paywall blocked posts the operator wasn't paying to see —
|
||||||
|
# operator-flagged 2026-05-31. Now: if no source-level error
|
||||||
|
# category fired AND tier-gated warnings are present, classify
|
||||||
|
# as TIER_LIMITED regardless of return code. Same priority order
|
||||||
|
# as before (auth/rate/access/not_found/network/http still win).
|
||||||
|
if not has_actual_error:
|
||||||
tier_gated_lines = [
|
tier_gated_lines = [
|
||||||
line for line in combined.split("\n")
|
line for line in combined.split("\n")
|
||||||
if "][warning]" in line and "not allowed to view post" in line
|
if "][warning]" in line and "not allowed to view post" in line
|
||||||
@@ -632,13 +654,57 @@ class GalleryDLService:
|
|||||||
started_at=started_at, completed_at=completed_at,
|
started_at=started_at, completed_at=completed_at,
|
||||||
)
|
)
|
||||||
|
|
||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired as e:
|
||||||
duration = time.time() - start_time
|
duration = time.time() - start_time
|
||||||
log.error("Download timeout for %s/%s after %.1fs", artist_slug, platform, duration)
|
# subprocess.run(text=True) makes these str if non-None, but the
|
||||||
|
# caller may have raised TimeoutExpired manually with None or
|
||||||
|
# bytes (tests do); coerce both cases to str.
|
||||||
|
partial_stdout = e.stdout or ""
|
||||||
|
partial_stderr = e.stderr or ""
|
||||||
|
if isinstance(partial_stdout, bytes):
|
||||||
|
partial_stdout = partial_stdout.decode("utf-8", "replace")
|
||||||
|
if isinstance(partial_stderr, bytes):
|
||||||
|
partial_stderr = partial_stderr.decode("utf-8", "replace")
|
||||||
|
|
||||||
|
files_so_far = self._count_downloaded_files(partial_stdout)
|
||||||
|
written_so_far = [str(p) for p in self._written_paths(partial_stdout)]
|
||||||
|
stderr_lines = partial_stderr.strip().splitlines()
|
||||||
|
tail_hint = stderr_lines[-1] if stderr_lines else "no stderr output"
|
||||||
|
|
||||||
|
# If the partial output already shows a rate-limit pattern, the
|
||||||
|
# timeout was almost certainly gallery-dl spinning on retries —
|
||||||
|
# promote to RATE_LIMITED so _update_source_health stamps the
|
||||||
|
# platform cooldown (same code path as a clean-exit rate limit).
|
||||||
|
# Otherwise stay TIMEOUT and let the captured stdout/stderr +
|
||||||
|
# files_so_far tell the operator whether it was "lots of
|
||||||
|
# content" vs "stuck retrying" vs "hung silent".
|
||||||
|
combined = (partial_stdout + "\n" + partial_stderr).lower()
|
||||||
|
if any(p in combined for p in self.RATE_LIMIT_PATTERNS):
|
||||||
|
error_type = ErrorType.RATE_LIMITED
|
||||||
|
error_message = (
|
||||||
|
f"Rate-limited and never completed within "
|
||||||
|
f"{source_config.timeout}s ({files_so_far} files written)"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
error_type = ErrorType.TIMEOUT
|
||||||
|
error_message = (
|
||||||
|
f"Download timed out after {source_config.timeout}s — "
|
||||||
|
f"{files_so_far} file(s) written; last stderr: {tail_hint}"
|
||||||
|
)
|
||||||
|
|
||||||
|
log.error(
|
||||||
|
"Download timeout for %s/%s after %.1fs (%d files written, "
|
||||||
|
"last stderr: %s)",
|
||||||
|
artist_slug, platform, duration, files_so_far, tail_hint,
|
||||||
|
)
|
||||||
|
|
||||||
return DownloadResult(
|
return DownloadResult(
|
||||||
success=False, url=url, artist_slug=artist_slug, platform=platform,
|
success=False, url=url, artist_slug=artist_slug, platform=platform,
|
||||||
error_type=ErrorType.TIMEOUT,
|
files_downloaded=files_so_far,
|
||||||
error_message=f"Download timed out after {source_config.timeout} seconds",
|
written_paths=written_so_far,
|
||||||
|
stdout=partial_stdout, stderr=partial_stderr,
|
||||||
|
return_code=-1, # killed by timeout, no real exit code
|
||||||
|
error_type=error_type, error_message=error_message,
|
||||||
duration_seconds=duration,
|
duration_seconds=duration,
|
||||||
started_at=started_at,
|
started_at=started_at,
|
||||||
completed_at=datetime.now(UTC).isoformat(),
|
completed_at=datetime.now(UTC).isoformat(),
|
||||||
@@ -658,3 +724,64 @@ class GalleryDLService:
|
|||||||
Path(temp_config_path).unlink() # noqa: ASYNC240
|
Path(temp_config_path).unlink() # noqa: ASYNC240
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
async def verify(
|
||||||
|
self,
|
||||||
|
url: str,
|
||||||
|
artist_slug: str,
|
||||||
|
platform: str,
|
||||||
|
source_config: SourceConfig | None = None,
|
||||||
|
cookies_path: str | None = None,
|
||||||
|
auth_token: str | None = None,
|
||||||
|
timeout: float = 45.0, # noqa: ASYNC109 — subprocess.run timeout, not a coroutine deadline
|
||||||
|
) -> tuple[bool, str]:
|
||||||
|
"""Test that credentials authenticate against `url` WITHOUT
|
||||||
|
downloading anything. Runs gallery-dl in --simulate mode limited
|
||||||
|
to the first item; if auth is bad the extractor errors before it
|
||||||
|
can list, which _categorize_error flags as AUTH_ERROR. Returns
|
||||||
|
(ok, message). Used by the credential Verify button."""
|
||||||
|
if source_config is None:
|
||||||
|
source_config = SourceConfig()
|
||||||
|
config = self._build_config_for_source(platform, source_config, artist_slug)
|
||||||
|
if cookies_path:
|
||||||
|
config["extractor"]["cookies"] = cookies_path
|
||||||
|
if auth_token and platform == "discord":
|
||||||
|
config["extractor"].setdefault("discord", {})["token"] = auth_token
|
||||||
|
if auth_token and platform == "pixiv":
|
||||||
|
config["extractor"].setdefault("pixiv", {})["refresh-token"] = auth_token
|
||||||
|
|
||||||
|
with tempfile.NamedTemporaryFile(
|
||||||
|
mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
|
||||||
|
) as fh:
|
||||||
|
json.dump(config, fh, indent=2)
|
||||||
|
temp_config_path = fh.name
|
||||||
|
try:
|
||||||
|
cmd = [
|
||||||
|
sys.executable, "-m", "gallery_dl",
|
||||||
|
"--config", temp_config_path,
|
||||||
|
"--simulate", "--range", "1-1", "--verbose", url,
|
||||||
|
]
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
proc = await loop.run_in_executor(
|
||||||
|
None,
|
||||||
|
lambda: subprocess.run(
|
||||||
|
cmd, capture_output=True, text=True, timeout=timeout,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
etype, msg = self._categorize_error(proc.returncode, proc.stdout, proc.stderr)
|
||||||
|
if proc.returncode == 0 or etype == ErrorType.NO_NEW_CONTENT:
|
||||||
|
return True, "Credentials valid — the feed authenticated."
|
||||||
|
if etype == ErrorType.AUTH_ERROR:
|
||||||
|
return False, msg
|
||||||
|
# Network / not-found / rate-limit / unknown: inconclusive,
|
||||||
|
# not a definitive credential failure. Surface the reason.
|
||||||
|
return False, f"Could not confirm ({etype.value}): {msg}"
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
return False, f"Verification timed out after {timeout:.0f}s"
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
return False, f"Verification error: {exc}"
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
Path(temp_config_path).unlink() # noqa: ASYNC240
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|||||||
@@ -90,9 +90,27 @@ class TimelineBucket:
|
|||||||
count: int
|
count: int
|
||||||
|
|
||||||
|
|
||||||
def thumbnail_url(sha256_hex: str, mime: str) -> str:
|
def thumbnail_url(thumbnail_path: str | None, sha256_hex: str, mime: str) -> str:
|
||||||
# Quart serves /images/* via the frontend blueprint (FC-1); thumbnails go
|
"""Return the URL to fetch a thumbnail.
|
||||||
# under /images/thumbs/. The MIME determines the extension.
|
|
||||||
|
Prefers the stored thumbnail_path verbatim — Quart serves /images/*
|
||||||
|
1:1 from the volume (frontend.py:20-36), so the URL IS the disk
|
||||||
|
path. Falls back to deriving from (sha256, mime) only when the
|
||||||
|
record's thumbnail_path is NULL (thumbnailer hasn't run yet); that
|
||||||
|
URL will 404 until backfill catches it, same as before the path
|
||||||
|
was tracked.
|
||||||
|
|
||||||
|
Pre-2026-05-30 this was derived only from (sha256, mime), which
|
||||||
|
disagreed with the actual on-disk extension when the thumbnailer
|
||||||
|
chose its format from transparency rather than MIME — every PNG
|
||||||
|
source without alpha (extension was .jpg on disk) and every WebP
|
||||||
|
source with alpha (extension was .png on disk) silently 404'd
|
||||||
|
despite the thumbnail file existing.
|
||||||
|
"""
|
||||||
|
if thumbnail_path:
|
||||||
|
return thumbnail_path
|
||||||
|
# Fallback for records with no thumbnail recorded yet — preserves
|
||||||
|
# prior behavior (URL exists but 404s until backfill regenerates).
|
||||||
ext = ".png" if mime in ("image/png", "image/gif") else ".jpg"
|
ext = ".png" if mime in ("image/png", "image/gif") else ".jpg"
|
||||||
bucket = sha256_hex[:3]
|
bucket = sha256_hex[:3]
|
||||||
return f"/images/thumbs/{bucket}/{sha256_hex}{ext}"
|
return f"/images/thumbs/{bucket}/{sha256_hex}{ext}"
|
||||||
@@ -198,7 +216,7 @@ class GalleryService:
|
|||||||
created_at=record.created_at,
|
created_at=record.created_at,
|
||||||
effective_date=eff_date,
|
effective_date=eff_date,
|
||||||
posted_at=posted_at,
|
posted_at=posted_at,
|
||||||
thumbnail_url=thumbnail_url(record.sha256, record.mime),
|
thumbnail_url=thumbnail_url(record.thumbnail_path, record.sha256, record.mime),
|
||||||
artist=artists.get(record.id),
|
artist=artists.get(record.id),
|
||||||
)
|
)
|
||||||
for record, posted_at, eff_date in rows
|
for record, posted_at, eff_date in rows
|
||||||
@@ -306,7 +324,7 @@ class GalleryService:
|
|||||||
"integrity_status": record.integrity_status,
|
"integrity_status": record.integrity_status,
|
||||||
"created_at": record.created_at.isoformat(),
|
"created_at": record.created_at.isoformat(),
|
||||||
"posted_at": posted_at.isoformat() if posted_at else None,
|
"posted_at": posted_at.isoformat() if posted_at else None,
|
||||||
"thumbnail_url": thumbnail_url(record.sha256, record.mime),
|
"thumbnail_url": thumbnail_url(record.thumbnail_path, record.sha256, record.mime),
|
||||||
"image_url": f"/images/{record.path.split('/images/', 1)[-1]}",
|
"image_url": f"/images/{record.path.split('/images/', 1)[-1]}",
|
||||||
"artist": (
|
"artist": (
|
||||||
{"id": artist.id, "name": artist.name, "slug": artist.slug}
|
{"id": artist.id, "name": artist.name, "slug": artist.slug}
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ from ..models import (
|
|||||||
PostAttachment,
|
PostAttachment,
|
||||||
Source,
|
Source,
|
||||||
)
|
)
|
||||||
|
from ..utils import safe_probe
|
||||||
from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name
|
from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name
|
||||||
from ..utils.phash import compute_phash, find_similar
|
from ..utils.phash import compute_phash, find_similar
|
||||||
from ..utils.sidecar import find_sidecar, parse_sidecar
|
from ..utils.sidecar import find_sidecar, parse_sidecar
|
||||||
@@ -203,6 +204,32 @@ class Importer:
|
|||||||
(phash, width or 0, height or 0, image_id)
|
(phash, width or 0, height or 0, image_id)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _get_or_create(self, stmt, factory):
|
||||||
|
"""Race-safe find-or-create. Run `stmt` (scalar_one_or_none); if a
|
||||||
|
row exists, return it. Otherwise open a savepoint and INSERT
|
||||||
|
``factory()``; on IntegrityError (a concurrent worker inserted the
|
||||||
|
same row first) roll the savepoint back — NOT the outer transaction,
|
||||||
|
which would lose the surrounding scan's progress — and re-run `stmt`
|
||||||
|
(scalar_one) to return the row the other worker created.
|
||||||
|
|
||||||
|
Centralizes the pattern shared by _find_or_create_source,
|
||||||
|
_source_for_sidecar, and _find_or_create_post. The plain
|
||||||
|
SELECT-then-INSERT version lost races under the 5-min recovery sweep
|
||||||
|
(operator-flagged 2026-05-26)."""
|
||||||
|
existing = self.session.execute(stmt).scalar_one_or_none()
|
||||||
|
if existing is not None:
|
||||||
|
return existing
|
||||||
|
sp = self.session.begin_nested()
|
||||||
|
try:
|
||||||
|
row = factory()
|
||||||
|
self.session.add(row)
|
||||||
|
self.session.flush()
|
||||||
|
sp.commit()
|
||||||
|
return row
|
||||||
|
except IntegrityError:
|
||||||
|
sp.rollback()
|
||||||
|
return self.session.execute(stmt).scalar_one()
|
||||||
|
|
||||||
def _find_or_create_source(
|
def _find_or_create_source(
|
||||||
self, *, artist_id: int, platform: str, url: str,
|
self, *, artist_id: int, platform: str, url: str,
|
||||||
) -> Source:
|
) -> Source:
|
||||||
@@ -221,53 +248,57 @@ class Importer:
|
|||||||
and re-select — the concurrent op just created the row we
|
and re-select — the concurrent op just created the row we
|
||||||
wanted, so the second select will find it.
|
wanted, so the second select will find it.
|
||||||
"""
|
"""
|
||||||
existing = self.session.execute(
|
stmt = select(Source).where(
|
||||||
select(Source).where(
|
Source.artist_id == artist_id,
|
||||||
Source.artist_id == artist_id,
|
Source.platform == platform,
|
||||||
Source.platform == platform,
|
Source.url == url,
|
||||||
Source.url == url,
|
)
|
||||||
)
|
return self._get_or_create(
|
||||||
).scalar_one_or_none()
|
stmt,
|
||||||
if existing is not None:
|
lambda: Source(artist_id=artist_id, platform=platform, url=url),
|
||||||
return existing
|
)
|
||||||
sp = self.session.begin_nested()
|
|
||||||
try:
|
|
||||||
row = Source(artist_id=artist_id, platform=platform, url=url)
|
|
||||||
self.session.add(row)
|
|
||||||
self.session.flush()
|
|
||||||
sp.commit()
|
|
||||||
return row
|
|
||||||
except IntegrityError:
|
|
||||||
sp.rollback()
|
|
||||||
return self.session.execute(
|
|
||||||
select(Source).where(
|
|
||||||
Source.artist_id == artist_id,
|
|
||||||
Source.platform == platform,
|
|
||||||
Source.url == url,
|
|
||||||
)
|
|
||||||
).scalar_one()
|
|
||||||
|
|
||||||
def _source_for_sidecar(
|
def _source_for_sidecar(
|
||||||
self, *, artist_id: int, platform: str, artist_slug: str,
|
self, *, artist_id: int, platform: str, artist_slug: str,
|
||||||
) -> Source:
|
) -> Source:
|
||||||
"""Filesystem-import sidecar Source resolver.
|
"""Sidecar-import Source resolver. Used by both filesystem imports
|
||||||
|
and gallery-dl downloads (both write sidecar JSON, both flow through
|
||||||
|
_apply_sidecar / _capture_attachment).
|
||||||
|
|
||||||
Source represents a subscription feed (one per artist+platform — the
|
Source represents a subscription feed (one per artist+platform — the
|
||||||
gallery-dl URL polled by the FC-3 downloader). The filesystem importer
|
URL polled by the FC-3 downloader). The filesystem importer used to
|
||||||
used to call _find_or_create_source(url=sd.post_url), which created
|
call _find_or_create_source(url=sd.post_url), creating one Source
|
||||||
one Source row per post URL — 100s of junk Sources per artist, all
|
row per post URL — 100s of junk Sources per artist, all with
|
||||||
with enabled=True, polluting the artist detail page and tricking the
|
enabled=True, polluting the artist detail page and tricking the
|
||||||
subscription checker into trying to poll patreon post URLs as feeds.
|
subscription checker into trying to poll patreon post URLs as feeds.
|
||||||
Operator-flagged 2026-05-26.
|
Operator-flagged 2026-05-26; consolidated via alembic 0022.
|
||||||
|
|
||||||
New behaviour: if any Source row exists for (artist_id, platform),
|
Resolution order: prefer a real (non-sidecar) Source over a
|
||||||
reuse it regardless of its URL — the artist's real subscription Source
|
synthetic anchor. When alembic 0022 ran, it may have rewritten
|
||||||
(created by the downloader / extension / UI) is the canonical
|
per-post Sources into `sidecar:<platform>:<slug>` synthetic
|
||||||
attachment point for filesystem-imported posts. If none exists, create
|
anchors. If the operator later added the real subscription, both
|
||||||
ONE synthetic anchor with url='sidecar:<platform>:<artist_slug>' and
|
rows now coexist. A naive `ORDER BY id ASC LIMIT 1` lookup would
|
||||||
enabled=False (so the subscription checker doesn't poll it).
|
pick the older synthetic and silently attach every gallery-dl
|
||||||
|
download to the wrong Source — operator-flagged 2026-05-31 after
|
||||||
|
the Subscriptions UI surfaced the phantom anchors. Pick the real
|
||||||
|
one when one exists; fall back to the synthetic; only create a
|
||||||
|
new synthetic when nothing exists for (artist, platform).
|
||||||
"""
|
"""
|
||||||
existing = self.session.execute(
|
real_stmt = (
|
||||||
|
select(Source)
|
||||||
|
.where(
|
||||||
|
Source.artist_id == artist_id,
|
||||||
|
Source.platform == platform,
|
||||||
|
~Source.url.like("sidecar:%"),
|
||||||
|
)
|
||||||
|
.order_by(Source.id.asc())
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
real = self.session.execute(real_stmt).scalar_one_or_none()
|
||||||
|
if real is not None:
|
||||||
|
return real
|
||||||
|
|
||||||
|
any_stmt = (
|
||||||
select(Source)
|
select(Source)
|
||||||
.where(
|
.where(
|
||||||
Source.artist_id == artist_id,
|
Source.artist_id == artist_id,
|
||||||
@@ -275,33 +306,16 @@ class Importer:
|
|||||||
)
|
)
|
||||||
.order_by(Source.id.asc())
|
.order_by(Source.id.asc())
|
||||||
.limit(1)
|
.limit(1)
|
||||||
).scalar_one_or_none()
|
)
|
||||||
if existing is not None:
|
return self._get_or_create(
|
||||||
return existing
|
any_stmt,
|
||||||
synthetic_url = f"sidecar:{platform}:{artist_slug}"
|
lambda: Source(
|
||||||
sp = self.session.begin_nested()
|
|
||||||
try:
|
|
||||||
row = Source(
|
|
||||||
artist_id=artist_id,
|
artist_id=artist_id,
|
||||||
platform=platform,
|
platform=platform,
|
||||||
url=synthetic_url,
|
url=f"sidecar:{platform}:{artist_slug}",
|
||||||
enabled=False,
|
enabled=False,
|
||||||
)
|
),
|
||||||
self.session.add(row)
|
)
|
||||||
self.session.flush()
|
|
||||||
sp.commit()
|
|
||||||
return row
|
|
||||||
except IntegrityError:
|
|
||||||
sp.rollback()
|
|
||||||
return self.session.execute(
|
|
||||||
select(Source)
|
|
||||||
.where(
|
|
||||||
Source.artist_id == artist_id,
|
|
||||||
Source.platform == platform,
|
|
||||||
)
|
|
||||||
.order_by(Source.id.asc())
|
|
||||||
.limit(1)
|
|
||||||
).scalar_one()
|
|
||||||
|
|
||||||
def _find_or_create_post(
|
def _find_or_create_post(
|
||||||
self, *, source_id: int, external_post_id: str,
|
self, *, source_id: int, external_post_id: str,
|
||||||
@@ -309,29 +323,14 @@ class Importer:
|
|||||||
"""Race-safe find-or-create on `post` keyed by
|
"""Race-safe find-or-create on `post` keyed by
|
||||||
(source_id, external_post_id). Mirrors `_find_or_create_source`
|
(source_id, external_post_id). Mirrors `_find_or_create_source`
|
||||||
— same savepoint + IntegrityError-recovery pattern."""
|
— same savepoint + IntegrityError-recovery pattern."""
|
||||||
existing = self.session.execute(
|
stmt = select(Post).where(
|
||||||
select(Post).where(
|
Post.source_id == source_id,
|
||||||
Post.source_id == source_id,
|
Post.external_post_id == external_post_id,
|
||||||
Post.external_post_id == external_post_id,
|
)
|
||||||
)
|
return self._get_or_create(
|
||||||
).scalar_one_or_none()
|
stmt,
|
||||||
if existing is not None:
|
lambda: Post(source_id=source_id, external_post_id=external_post_id),
|
||||||
return existing
|
)
|
||||||
sp = self.session.begin_nested()
|
|
||||||
try:
|
|
||||||
row = Post(source_id=source_id, external_post_id=external_post_id)
|
|
||||||
self.session.add(row)
|
|
||||||
self.session.flush()
|
|
||||||
sp.commit()
|
|
||||||
return row
|
|
||||||
except IntegrityError:
|
|
||||||
sp.rollback()
|
|
||||||
return self.session.execute(
|
|
||||||
select(Post).where(
|
|
||||||
Post.source_id == source_id,
|
|
||||||
Post.external_post_id == external_post_id,
|
|
||||||
)
|
|
||||||
).scalar_one()
|
|
||||||
|
|
||||||
def import_one(self, source: Path) -> ImportResult:
|
def import_one(self, source: Path) -> ImportResult:
|
||||||
"""Dispatch by kind. Media → normal pipeline. Archive → extract
|
"""Dispatch by kind. Media → normal pipeline. Archive → extract
|
||||||
@@ -407,6 +406,29 @@ class Importer:
|
|||||||
return ImportResult(status="attached")
|
return ImportResult(status="attached")
|
||||||
|
|
||||||
def _import_archive(self, source: Path) -> ImportResult:
|
def _import_archive(self, source: Path) -> ImportResult:
|
||||||
|
# Layer-3 isolation: bomb-size guard + integrity test in a
|
||||||
|
# spawned child BEFORE extracting in this process. A
|
||||||
|
# decompression bomb or a native-lib crash on a malformed
|
||||||
|
# archive is contained to the child; we reject the file cleanly
|
||||||
|
# instead of OOMing/segfaulting the import worker. extract_archive
|
||||||
|
# is already fail-soft for plain exceptions, so this only adds
|
||||||
|
# the hard-crash protection.
|
||||||
|
probe = safe_probe.probe_archive(source)
|
||||||
|
if not probe.ok:
|
||||||
|
if probe.crashed:
|
||||||
|
return ImportResult(
|
||||||
|
status="failed",
|
||||||
|
error=f"archive probe crashed/timed out: {probe.reason}",
|
||||||
|
)
|
||||||
|
# Clean rejection (bomb cap exceeded, integrity mismatch):
|
||||||
|
# still preserve the archive file itself as an attachment so
|
||||||
|
# nothing silently vanishes, matching extract_archive's
|
||||||
|
# fail-soft contract.
|
||||||
|
artist = self._resolve_artist(source)
|
||||||
|
post = self._post_for_sidecar(source, artist)
|
||||||
|
self._capture_attachment(source, post=post, artist=artist, resolved=True)
|
||||||
|
return ImportResult(status="attached")
|
||||||
|
|
||||||
artist = self._resolve_artist(source)
|
artist = self._resolve_artist(source)
|
||||||
post = self._post_for_sidecar(source, artist)
|
post = self._post_for_sidecar(source, artist)
|
||||||
member_ids: list[int] = []
|
member_ids: list[int] = []
|
||||||
@@ -446,7 +468,25 @@ class Importer:
|
|||||||
# Compute file dimensions (images only) and apply filters.
|
# Compute file dimensions (images only) and apply filters.
|
||||||
width = height = None
|
width = height = None
|
||||||
has_alpha = False
|
has_alpha = False
|
||||||
if not is_video(source):
|
if is_video(source):
|
||||||
|
# Layer-3 isolation: validate the container via ffprobe (a
|
||||||
|
# separate process) before the rest of the pipeline touches
|
||||||
|
# it. A corrupt video that would crash a decoder is rejected
|
||||||
|
# cleanly here, and we capture width/height for free (the
|
||||||
|
# importer didn't previously record video dimensions).
|
||||||
|
probe = safe_probe.probe_video(source)
|
||||||
|
if not probe.ok:
|
||||||
|
if probe.crashed:
|
||||||
|
return ImportResult(
|
||||||
|
status="failed",
|
||||||
|
error=f"video probe crashed/timed out: {probe.reason}",
|
||||||
|
)
|
||||||
|
return ImportResult(
|
||||||
|
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||||
|
error=probe.reason,
|
||||||
|
)
|
||||||
|
width, height = probe.width, probe.height
|
||||||
|
else:
|
||||||
try:
|
try:
|
||||||
with Image.open(source) as im:
|
with Image.open(source) as im:
|
||||||
im.verify()
|
im.verify()
|
||||||
|
|||||||
@@ -1,10 +0,0 @@
|
|||||||
"""FC-5 migration tooling.
|
|
||||||
|
|
||||||
One module per concern (gs/ir/overlap/ml_queue/verify/cleanup).
|
|
||||||
Each migrator returns a counts dict; the run_migration task wires
|
|
||||||
that dict into MigrationRun.counts so the UI polling shows progress.
|
|
||||||
|
|
||||||
backup + rollback were retired in FC-3h (2026-05-24); first-class
|
|
||||||
backup lives at backend/app/services/backup_service.py and exposes
|
|
||||||
its own /api/system/backup/* surface.
|
|
||||||
"""
|
|
||||||
@@ -1,182 +0,0 @@
|
|||||||
"""Targeted cleanup migrator: delete every image attributed to one Artist.
|
|
||||||
|
|
||||||
Built for the IR-migration rescue case where the filesystem scan derived
|
|
||||||
a bogus 'imagerepo' artist from a mismatched bind-mount layout. Every
|
|
||||||
image attributed to that artist (40k+ rows) needs to be removed — DB
|
|
||||||
rows, original files under `/images/<bucket>/...`, and thumbnails under
|
|
||||||
`/images/thumbs/...` — before the operator remounts and re-scans.
|
|
||||||
|
|
||||||
CASCADE handles image_tag, image_provenance, series_page, and
|
|
||||||
tag_suggestion_rejection child rows; import_task.result_image_id is
|
|
||||||
SET NULL by FK. We also delete ImportTask rows whose source_path starts
|
|
||||||
with the (still-existing) IR scan prefix so the next scan isn't fooled
|
|
||||||
by them.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from sqlalchemy import delete, func, select
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
from ...models import Artist, ImageRecord, ImportBatch, ImportTask
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_BATCH_SIZE = 500
|
|
||||||
|
|
||||||
|
|
||||||
def _zero_counts() -> dict:
|
|
||||||
return {
|
|
||||||
"rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0,
|
|
||||||
"files_copied": 0, "bytes_copied": 0, "conflicts": 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _thumb_path(images_root: Path, sha256_hex: str) -> tuple[Path, Path]:
|
|
||||||
"""Return both possible thumbnail paths (.jpg and .png). We try both
|
|
||||||
because the extension is chosen at generate-time based on the source
|
|
||||||
image's mode (alpha → .png, otherwise → .jpg)."""
|
|
||||||
bucket = sha256_hex[:3]
|
|
||||||
base = images_root / "thumbs" / bucket / sha256_hex
|
|
||||||
return base.with_suffix(".jpg"), base.with_suffix(".png")
|
|
||||||
|
|
||||||
|
|
||||||
def _delete_file(path: Path) -> bool:
|
|
||||||
"""Best-effort unlink; True if the file was actually removed."""
|
|
||||||
try:
|
|
||||||
path.unlink(missing_ok=True)
|
|
||||||
return True
|
|
||||||
except OSError as exc:
|
|
||||||
log.warning("cleanup: failed to unlink %s: %s", path, exc)
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
async def cleanup_artist_async(
|
|
||||||
db: AsyncSession,
|
|
||||||
*,
|
|
||||||
slug: str,
|
|
||||||
images_root: Path | None = None,
|
|
||||||
dry_run: bool = False,
|
|
||||||
source_path_prefix: str | None = None,
|
|
||||||
) -> dict:
|
|
||||||
"""Delete every image attributed to the Artist with this slug,
|
|
||||||
along with the artist row itself and any associated import tasks.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
slug: artist.slug to target (e.g. 'imagerepo').
|
|
||||||
images_root: defaults to /images.
|
|
||||||
dry_run: skip filesystem + DB writes; still walk rows for counts.
|
|
||||||
source_path_prefix: if set, ImportTask rows whose source_path
|
|
||||||
starts with this string are deleted too (use the IR scan
|
|
||||||
mount prefix, e.g. '/import/imagerepo').
|
|
||||||
"""
|
|
||||||
root = images_root if images_root is not None else Path("/images")
|
|
||||||
|
|
||||||
artist = (await db.execute(
|
|
||||||
select(Artist).where(Artist.slug == slug)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if artist is None:
|
|
||||||
raise ValueError(f"no Artist with slug={slug!r}")
|
|
||||||
|
|
||||||
artist_id = artist.id
|
|
||||||
artist_name = artist.name
|
|
||||||
|
|
||||||
total_images = (await db.execute(
|
|
||||||
select(func.count(ImageRecord.id)).where(ImageRecord.artist_id == artist_id)
|
|
||||||
)).scalar_one()
|
|
||||||
|
|
||||||
counts = _zero_counts()
|
|
||||||
files_deleted = 0
|
|
||||||
thumbs_deleted = 0
|
|
||||||
images_deleted = 0
|
|
||||||
|
|
||||||
# Batched delete loop. CASCADE handles image_tag, image_provenance,
|
|
||||||
# series_page, tag_suggestion_rejection. import_task.result_image_id
|
|
||||||
# is SET NULL by FK.
|
|
||||||
while True:
|
|
||||||
rows = (await db.execute(
|
|
||||||
select(ImageRecord.id, ImageRecord.path, ImageRecord.sha256)
|
|
||||||
.where(ImageRecord.artist_id == artist_id)
|
|
||||||
.limit(_BATCH_SIZE)
|
|
||||||
)).all()
|
|
||||||
if not rows:
|
|
||||||
break
|
|
||||||
|
|
||||||
ids = [r.id for r in rows]
|
|
||||||
counts["rows_processed"] += len(ids)
|
|
||||||
|
|
||||||
if not dry_run:
|
|
||||||
for r in rows:
|
|
||||||
if r.path:
|
|
||||||
if _delete_file(Path(r.path)):
|
|
||||||
files_deleted += 1
|
|
||||||
if r.sha256:
|
|
||||||
jpg, png = _thumb_path(root, r.sha256)
|
|
||||||
if _delete_file(jpg):
|
|
||||||
thumbs_deleted += 1
|
|
||||||
if _delete_file(png):
|
|
||||||
thumbs_deleted += 1
|
|
||||||
|
|
||||||
await db.execute(
|
|
||||||
delete(ImageRecord).where(ImageRecord.id.in_(ids))
|
|
||||||
)
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
images_deleted += len(ids)
|
|
||||||
|
|
||||||
if dry_run:
|
|
||||||
# Nothing was actually deleted from the DB; bail after one
|
|
||||||
# pass so we don't loop forever.
|
|
||||||
break
|
|
||||||
|
|
||||||
import_tasks_deleted = 0
|
|
||||||
if source_path_prefix and not dry_run:
|
|
||||||
# Delete ImportTask rows whose source_path is under the bad mount
|
|
||||||
# prefix. These are mostly orphaned now (result_image_id was set
|
|
||||||
# NULL by CASCADE) but their presence still blocks the
|
|
||||||
# idempotency check in scan_directory if the operator remounts
|
|
||||||
# the same prefix.
|
|
||||||
like_pattern = source_path_prefix.rstrip("/") + "/%"
|
|
||||||
result = await db.execute(
|
|
||||||
delete(ImportTask).where(ImportTask.source_path.like(like_pattern))
|
|
||||||
)
|
|
||||||
import_tasks_deleted = result.rowcount or 0
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
# Sweep ImportBatch rows that are now empty.
|
|
||||||
empty_batches_deleted = 0
|
|
||||||
if not dry_run:
|
|
||||||
empty_batch_ids = (await db.execute(
|
|
||||||
select(ImportBatch.id).where(
|
|
||||||
~select(ImportTask.id)
|
|
||||||
.where(ImportTask.batch_id == ImportBatch.id)
|
|
||||||
.exists()
|
|
||||||
)
|
|
||||||
)).scalars().all()
|
|
||||||
if empty_batch_ids:
|
|
||||||
result = await db.execute(
|
|
||||||
delete(ImportBatch).where(ImportBatch.id.in_(empty_batch_ids))
|
|
||||||
)
|
|
||||||
empty_batches_deleted = result.rowcount or 0
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
# Finally, the artist row.
|
|
||||||
if not dry_run:
|
|
||||||
await db.execute(delete(Artist).where(Artist.id == artist_id))
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
return {
|
|
||||||
"counts": counts,
|
|
||||||
"artist": {"id": artist_id, "name": artist_name, "slug": slug},
|
|
||||||
"summary": {
|
|
||||||
"images_targeted": total_images,
|
|
||||||
"images_deleted": images_deleted,
|
|
||||||
"files_deleted": files_deleted,
|
|
||||||
"thumbs_deleted": thumbs_deleted,
|
|
||||||
"import_tasks_deleted": import_tasks_deleted,
|
|
||||||
"empty_batches_deleted": empty_batches_deleted,
|
|
||||||
"dry_run": dry_run,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
@@ -1,126 +0,0 @@
|
|||||||
"""GallerySubscriber export → FabledCurator ingest.
|
|
||||||
|
|
||||||
Reads a parsed gallerysubscriber-export-v1.json dict (no DB connection
|
|
||||||
to GS). Creates Artist (from subscriptions) + Source (nested under each
|
|
||||||
subscription) + Credential (re-encrypted with FC's key). Idempotent on
|
|
||||||
natural keys: Artist.slug, (artist_id, platform, url), Credential.platform.
|
|
||||||
|
|
||||||
Credentials arrive plaintext in the export — GS's export script
|
|
||||||
decrypts using GS's Fernet key in GS's own process. FC re-encrypts
|
|
||||||
with FC's CredentialCrypto.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
from ...models import Artist, Credential, Source
|
|
||||||
from ...utils.slug import slugify
|
|
||||||
from ..credential_crypto import CredentialCrypto
|
|
||||||
|
|
||||||
|
|
||||||
def _zero_counts() -> dict:
|
|
||||||
return {
|
|
||||||
"rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0,
|
|
||||||
"files_copied": 0, "bytes_copied": 0, "conflicts": 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async def migrate_async(
|
|
||||||
db: AsyncSession,
|
|
||||||
*,
|
|
||||||
data: dict,
|
|
||||||
fc_crypto: CredentialCrypto | None = None,
|
|
||||||
dry_run: bool = False,
|
|
||||||
) -> dict:
|
|
||||||
"""Ingest a parsed gallerysubscriber-export-v1.json dict."""
|
|
||||||
if data.get("source_app") != "gallerysubscriber":
|
|
||||||
raise ValueError("export source_app must be 'gallerysubscriber'")
|
|
||||||
if data.get("schema_version") != 1:
|
|
||||||
raise ValueError(f"unsupported schema_version: {data.get('schema_version')}")
|
|
||||||
|
|
||||||
counts = _zero_counts()
|
|
||||||
|
|
||||||
# Phase 1: subscriptions → Artist; nested sources within each.
|
|
||||||
for sub in data.get("subscriptions", []):
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
slug = slugify(sub["name"])
|
|
||||||
artist = (await db.execute(
|
|
||||||
select(Artist).where(Artist.slug == slug)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if artist is None:
|
|
||||||
if dry_run:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
# Continue to nested sources, but they can't link without an artist row.
|
|
||||||
continue
|
|
||||||
notes = json.dumps(sub.get("metadata"), indent=2) if sub.get("metadata") else None
|
|
||||||
artist = Artist(
|
|
||||||
name=sub["name"], slug=slug,
|
|
||||||
is_subscription=True,
|
|
||||||
auto_check=bool(sub.get("enabled", True)),
|
|
||||||
notes=notes,
|
|
||||||
)
|
|
||||||
db.add(artist)
|
|
||||||
await db.flush()
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
else:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
|
|
||||||
# Nested sources under this subscription.
|
|
||||||
for src in sub.get("sources", []):
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(Source).where(
|
|
||||||
Source.artist_id == artist.id,
|
|
||||||
Source.platform == src["platform"],
|
|
||||||
Source.url == src["url"],
|
|
||||||
)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
if dry_run:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
continue
|
|
||||||
db.add(Source(
|
|
||||||
artist_id=artist.id,
|
|
||||||
platform=src["platform"],
|
|
||||||
url=src["url"],
|
|
||||||
enabled=bool(src.get("enabled", True)),
|
|
||||||
check_interval_override=src.get("check_interval"),
|
|
||||||
config_overrides=src.get("metadata") or {},
|
|
||||||
))
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
|
|
||||||
# Phase 2: credentials.
|
|
||||||
for cred in data.get("credentials", []):
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(Credential).where(Credential.platform == cred["platform"])
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
if dry_run:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
continue
|
|
||||||
if fc_crypto is None:
|
|
||||||
# Without a crypto helper we can't encrypt — skip rather than
|
|
||||||
# store plaintext.
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
counts["conflicts"] += 1
|
|
||||||
continue
|
|
||||||
encrypted = fc_crypto.encrypt(cred["plaintext"])
|
|
||||||
db.add(Credential(
|
|
||||||
platform=cred["platform"],
|
|
||||||
credential_type=cred.get("credential_type") or "cookies",
|
|
||||||
encrypted_blob=encrypted,
|
|
||||||
expires_at=cred.get("expires_at"),
|
|
||||||
))
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
|
|
||||||
if not dry_run:
|
|
||||||
await db.commit()
|
|
||||||
return counts
|
|
||||||
@@ -1,146 +0,0 @@
|
|||||||
"""ImageRepo export → FabledCurator ingest.
|
|
||||||
|
|
||||||
Reads a parsed imagerepo-export-v1.json dict (no DB connection to IR).
|
|
||||||
Creates Tag rows (skipping artist/post kinds, resolving fandom_name to
|
|
||||||
FK). Writes the per-image-sha256 artist assignments + tag associations
|
|
||||||
+ series page assignments to /images/_migration_state/ir_tag_manifest.json
|
|
||||||
so tag_apply.py can join them to ImageRecord rows AFTER the operator
|
|
||||||
runs FC's filesystem scan.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
from ...models import Tag, TagKind
|
|
||||||
|
|
||||||
_SKIP_KINDS = frozenset({"artist", "post"})
|
|
||||||
_MIGRATION_STATE_DIRNAME = "_migration_state"
|
|
||||||
_IR_MANIFEST_FILENAME = "ir_tag_manifest.json"
|
|
||||||
|
|
||||||
|
|
||||||
def _zero_counts() -> dict:
|
|
||||||
return {
|
|
||||||
"rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0,
|
|
||||||
"files_copied": 0, "bytes_copied": 0, "conflicts": 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def manifest_path(images_root: Path | None = None) -> Path:
|
|
||||||
root = images_root if images_root is not None else Path("/images")
|
|
||||||
p = root / _MIGRATION_STATE_DIRNAME
|
|
||||||
p.mkdir(parents=True, exist_ok=True)
|
|
||||||
return p / _IR_MANIFEST_FILENAME
|
|
||||||
|
|
||||||
|
|
||||||
async def _resolve_fandom_id(
|
|
||||||
db: AsyncSession, fandom_name: str | None, dry_run: bool,
|
|
||||||
) -> int | None:
|
|
||||||
"""Find-or-create a fandom-kind Tag by name."""
|
|
||||||
if not fandom_name:
|
|
||||||
return None
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(Tag).where(Tag.name == fandom_name, Tag.kind == "fandom")
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
return existing.id
|
|
||||||
if dry_run:
|
|
||||||
return None
|
|
||||||
t = Tag(name=fandom_name, kind=TagKind.fandom)
|
|
||||||
db.add(t)
|
|
||||||
await db.flush()
|
|
||||||
return t.id
|
|
||||||
|
|
||||||
|
|
||||||
async def migrate_async(
|
|
||||||
db: AsyncSession,
|
|
||||||
*,
|
|
||||||
data: dict,
|
|
||||||
images_root: Path | None = None,
|
|
||||||
dry_run: bool = False,
|
|
||||||
) -> dict:
|
|
||||||
"""Ingest a parsed imagerepo-export-v1.json dict.
|
|
||||||
|
|
||||||
Creates Tag rows + writes the IR tag manifest file. Tag-to-image
|
|
||||||
binding happens later in tag_apply.py (after FC's filesystem scan
|
|
||||||
populates image_record.sha256 → id).
|
|
||||||
"""
|
|
||||||
if data.get("source_app") != "imagerepo":
|
|
||||||
raise ValueError("export source_app must be 'imagerepo'")
|
|
||||||
if data.get("schema_version") not in (1, 2):
|
|
||||||
raise ValueError(f"unsupported schema_version: {data.get('schema_version')}")
|
|
||||||
|
|
||||||
counts = _zero_counts()
|
|
||||||
|
|
||||||
# Phase 1: tags (skip artist + post kinds; resolve fandom_name → fandom_id).
|
|
||||||
# First pass: create all fandom-kind tags so they're available for FK resolution.
|
|
||||||
for tag in data.get("tags", []):
|
|
||||||
kind = tag.get("kind") or "general"
|
|
||||||
if kind != "fandom":
|
|
||||||
continue
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(Tag).where(Tag.name == tag["name"], Tag.kind == "fandom")
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
if dry_run:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
continue
|
|
||||||
db.add(Tag(name=tag["name"], kind=TagKind.fandom))
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
if not dry_run:
|
|
||||||
await db.flush()
|
|
||||||
|
|
||||||
# Second pass: every other kind.
|
|
||||||
for tag in data.get("tags", []):
|
|
||||||
kind_str = tag.get("kind") or "general"
|
|
||||||
if kind_str in _SKIP_KINDS:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
if kind_str == "fandom":
|
|
||||||
continue # handled above
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
try:
|
|
||||||
kind = TagKind(kind_str)
|
|
||||||
except ValueError:
|
|
||||||
kind = TagKind.general
|
|
||||||
fandom_id = await _resolve_fandom_id(db, tag.get("fandom_name"), dry_run)
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(Tag).where(Tag.name == tag["name"], Tag.kind == kind)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
if dry_run:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
continue
|
|
||||||
db.add(Tag(name=tag["name"], kind=kind, fandom_id=fandom_id))
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
|
|
||||||
if not dry_run:
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
# Phase 2: write the per-image manifest for tag_apply.py to consume later.
|
|
||||||
# schema_version 2 (added 2026-05-24) carries `image_posts` for
|
|
||||||
# Post + Source + ImageProvenance restore; schema 1 manifests
|
|
||||||
# without it stay valid (tag_apply treats the missing field as []).
|
|
||||||
manifest = {
|
|
||||||
"schema_version": data.get("schema_version", 1),
|
|
||||||
"image_artist_assignments": data.get("image_artist_assignments", []),
|
|
||||||
"image_tag_associations": data.get("image_tag_associations", []),
|
|
||||||
"series_pages": data.get("series_pages", []),
|
|
||||||
"image_posts": data.get("image_posts", []),
|
|
||||||
}
|
|
||||||
counts["rows_processed"] += len(manifest["image_artist_assignments"])
|
|
||||||
counts["rows_processed"] += len(manifest["image_tag_associations"])
|
|
||||||
counts["rows_processed"] += len(manifest["series_pages"])
|
|
||||||
counts["rows_processed"] += len(manifest["image_posts"])
|
|
||||||
if not dry_run:
|
|
||||||
manifest_path(images_root).write_text(json.dumps(manifest, indent=2))
|
|
||||||
|
|
||||||
return counts
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
"""Queue every migrated image_record with no embedding for ML re-processing."""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
from ...models import ImageRecord
|
|
||||||
|
|
||||||
|
|
||||||
async def queue_all_unprocessed_async(db: AsyncSession) -> int:
|
|
||||||
"""Find every ImageRecord with siglip_embedding IS NULL, fire
|
|
||||||
tag_and_embed.delay(id) for each. Returns count queued.
|
|
||||||
"""
|
|
||||||
from ...tasks.ml import tag_and_embed
|
|
||||||
|
|
||||||
rows = (await db.execute(
|
|
||||||
select(ImageRecord.id).where(ImageRecord.siglip_embedding.is_(None))
|
|
||||||
)).scalars().all()
|
|
||||||
for image_id in rows:
|
|
||||||
tag_and_embed.delay(image_id)
|
|
||||||
return len(rows)
|
|
||||||
@@ -1,368 +0,0 @@
|
|||||||
"""Apply the IR tag manifest after FC's filesystem scan.
|
|
||||||
|
|
||||||
Reads /images/_migration_state/ir_tag_manifest.json and joins each entry
|
|
||||||
to an ImageRecord row by sha256 (which exists after the operator runs
|
|
||||||
FC's filesystem scan over the mounted IR images dir).
|
|
||||||
|
|
||||||
- image_artist_assignments → ImageRecord.artist_id (find_or_create Artist by slug).
|
|
||||||
- image_tag_associations → image_tag insert (idempotent).
|
|
||||||
- series_pages → series_page insert (idempotent on image_id unique).
|
|
||||||
- image_posts (schema v2) → Source + Post + ImageProvenance restore.
|
|
||||||
|
|
||||||
Unmatched sha256s are logged into the result's `unmatched` list so the
|
|
||||||
Celery task can drop them into MigrationRun.metadata for the operator
|
|
||||||
to inspect.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
from datetime import datetime
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
from ...models import (
|
|
||||||
Artist,
|
|
||||||
ImageProvenance,
|
|
||||||
ImageRecord,
|
|
||||||
Post,
|
|
||||||
SeriesPage,
|
|
||||||
Source,
|
|
||||||
Tag,
|
|
||||||
TagKind,
|
|
||||||
image_tag,
|
|
||||||
)
|
|
||||||
from ...utils.slug import slugify
|
|
||||||
from .ir_ingest import manifest_path
|
|
||||||
|
|
||||||
# Per-platform artist-profile URL — used as Source.url when restoring
|
|
||||||
# IR PostMetadata into FC. Must cover every platform that
|
|
||||||
# backend/app/services/extension_service.py:_PLATFORM_PATTERNS
|
|
||||||
# recognizes; an entry missing here silently drops ALL PostMetadata for
|
|
||||||
# that platform during phase 4 (operator hit this 2026-05-25:
|
|
||||||
# DeviantArt + Pixiv posts in the IR migration produced empty
|
|
||||||
# ImageProvenance because they fell through this table).
|
|
||||||
#
|
|
||||||
# Pixiv caveat: the real profile URL takes a numeric user_id
|
|
||||||
# (https://www.pixiv.net/users/12345), but IR's PostMetadata.artist
|
|
||||||
# stores the display name not the id. We use the slugified name here
|
|
||||||
# so we preserve the artist→post→image linkage; the resulting Source.url
|
|
||||||
# won't resolve in a browser and the operator may want to manually fix
|
|
||||||
# it via Settings → Subscriptions once the migration lands.
|
|
||||||
_PLATFORM_PROFILE_URL = {
|
|
||||||
"patreon": "https://www.patreon.com/{slug}",
|
|
||||||
"subscribestar": "https://www.subscribestar.com/{slug}",
|
|
||||||
"hentaifoundry": "https://www.hentai-foundry.com/user/{slug}",
|
|
||||||
"deviantart": "https://www.deviantart.com/{slug}",
|
|
||||||
"pixiv": "https://www.pixiv.net/users/{slug}",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _profile_url(platform: str, artist_slug: str) -> str | None:
|
|
||||||
fmt = _PLATFORM_PROFILE_URL.get(platform)
|
|
||||||
return fmt.format(slug=artist_slug) if fmt else None
|
|
||||||
|
|
||||||
|
|
||||||
async def _find_or_create_source(
|
|
||||||
db: AsyncSession, *, artist_id: int, platform: str, url: str, dry_run: bool,
|
|
||||||
) -> int | None:
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(Source.id).where(
|
|
||||||
Source.artist_id == artist_id,
|
|
||||||
Source.platform == platform,
|
|
||||||
Source.url == url,
|
|
||||||
)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
return existing
|
|
||||||
if dry_run:
|
|
||||||
return None
|
|
||||||
s = Source(artist_id=artist_id, platform=platform, url=url, enabled=False)
|
|
||||||
db.add(s)
|
|
||||||
await db.flush()
|
|
||||||
return s.id
|
|
||||||
|
|
||||||
|
|
||||||
async def _find_or_create_post(
|
|
||||||
db: AsyncSession, *,
|
|
||||||
source_id: int, external_post_id: str,
|
|
||||||
title: str | None, description: str | None, post_url: str | None,
|
|
||||||
post_date_iso: str | None, attachment_count: int, dry_run: bool,
|
|
||||||
) -> int | None:
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(Post.id).where(
|
|
||||||
Post.source_id == source_id,
|
|
||||||
Post.external_post_id == external_post_id,
|
|
||||||
)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
return existing
|
|
||||||
if dry_run:
|
|
||||||
return None
|
|
||||||
post_date = None
|
|
||||||
if post_date_iso:
|
|
||||||
post_date = datetime.fromisoformat(post_date_iso)
|
|
||||||
p = Post(
|
|
||||||
source_id=source_id,
|
|
||||||
external_post_id=external_post_id,
|
|
||||||
post_title=title,
|
|
||||||
description=description,
|
|
||||||
post_url=post_url,
|
|
||||||
post_date=post_date,
|
|
||||||
attachment_count=attachment_count,
|
|
||||||
raw_metadata={"migrated_from": "imagerepo"},
|
|
||||||
)
|
|
||||||
db.add(p)
|
|
||||||
await db.flush()
|
|
||||||
return p.id
|
|
||||||
|
|
||||||
|
|
||||||
async def _ensure_provenance(
|
|
||||||
db: AsyncSession, *,
|
|
||||||
image_id: int, post_id: int, source_id: int, dry_run: bool,
|
|
||||||
) -> bool:
|
|
||||||
"""Returns True if a new ImageProvenance row was inserted.
|
|
||||||
|
|
||||||
Also sets ImageRecord.primary_post_id to this post if the image
|
|
||||||
doesn't already have one — preserves any primary_post_id already
|
|
||||||
assigned at download time by the importer (don't clobber). This is
|
|
||||||
the linkage gallery_service.py uses to surface Post.post_date as
|
|
||||||
the image's effective date for sort/group/jump/neighbor nav.
|
|
||||||
"""
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(ImageProvenance.id).where(
|
|
||||||
ImageProvenance.image_record_id == image_id,
|
|
||||||
ImageProvenance.post_id == post_id,
|
|
||||||
ImageProvenance.source_id == source_id,
|
|
||||||
)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
|
|
||||||
# Whether-or-not the provenance row already exists, ensure the
|
|
||||||
# image's primary_post_id is set so the gallery date-coalesce works.
|
|
||||||
# Idempotent: only writes when currently NULL.
|
|
||||||
if not dry_run:
|
|
||||||
await db.execute(
|
|
||||||
ImageRecord.__table__.update()
|
|
||||||
.where(ImageRecord.id == image_id)
|
|
||||||
.where(ImageRecord.primary_post_id.is_(None))
|
|
||||||
.values(primary_post_id=post_id)
|
|
||||||
)
|
|
||||||
|
|
||||||
if existing is not None:
|
|
||||||
return False
|
|
||||||
if dry_run:
|
|
||||||
return True
|
|
||||||
db.add(ImageProvenance(
|
|
||||||
image_record_id=image_id, post_id=post_id, source_id=source_id,
|
|
||||||
))
|
|
||||||
await db.flush()
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def _zero_counts() -> dict:
|
|
||||||
return {
|
|
||||||
"rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0,
|
|
||||||
"files_copied": 0, "bytes_copied": 0, "conflicts": 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async def _ensure_artist_id(
|
|
||||||
db: AsyncSession, artist_name: str, dry_run: bool,
|
|
||||||
) -> int | None:
|
|
||||||
if not artist_name or not artist_name.strip():
|
|
||||||
return None
|
|
||||||
slug = slugify(artist_name)
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(Artist).where(Artist.slug == slug)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
return existing.id
|
|
||||||
if dry_run:
|
|
||||||
return None
|
|
||||||
a = Artist(name=artist_name, slug=slug, is_subscription=False)
|
|
||||||
db.add(a)
|
|
||||||
await db.flush()
|
|
||||||
return a.id
|
|
||||||
|
|
||||||
|
|
||||||
async def _resolve_tag_id(
|
|
||||||
db: AsyncSession, tag_name: str, tag_kind: str,
|
|
||||||
) -> int | None:
|
|
||||||
try:
|
|
||||||
kind = TagKind(tag_kind)
|
|
||||||
except ValueError:
|
|
||||||
kind = TagKind.general
|
|
||||||
row = (await db.execute(
|
|
||||||
select(Tag.id).where(Tag.name == tag_name, Tag.kind == kind)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
return row
|
|
||||||
|
|
||||||
|
|
||||||
async def _sha_to_image_id(db: AsyncSession, sha: str) -> int | None:
|
|
||||||
return (await db.execute(
|
|
||||||
select(ImageRecord.id).where(ImageRecord.sha256 == sha)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
|
|
||||||
|
|
||||||
async def apply_async(
|
|
||||||
db: AsyncSession,
|
|
||||||
*,
|
|
||||||
images_root: Path | None = None,
|
|
||||||
dry_run: bool = False,
|
|
||||||
) -> dict:
|
|
||||||
"""Apply the manifest. Returns counts + an `unmatched` list of sha256s."""
|
|
||||||
mf_path = manifest_path(images_root)
|
|
||||||
if not mf_path.exists():
|
|
||||||
raise FileNotFoundError(f"no IR tag manifest at {mf_path}")
|
|
||||||
|
|
||||||
manifest = json.loads(mf_path.read_text())
|
|
||||||
counts = _zero_counts()
|
|
||||||
unmatched: list[dict] = []
|
|
||||||
|
|
||||||
# 1. Artist assignments.
|
|
||||||
for entry in manifest.get("image_artist_assignments", []):
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
img_id = await _sha_to_image_id(db, entry["sha256"])
|
|
||||||
if img_id is None:
|
|
||||||
unmatched.append({"kind": "artist", **entry})
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
aid = await _ensure_artist_id(db, entry["artist_name"], dry_run)
|
|
||||||
if aid is None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
if dry_run:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
continue
|
|
||||||
img = await db.get(ImageRecord, img_id)
|
|
||||||
if img is not None and img.artist_id != aid:
|
|
||||||
img.artist_id = aid
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
else:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
|
|
||||||
# 2. Tag associations.
|
|
||||||
for entry in manifest.get("image_tag_associations", []):
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
img_id = await _sha_to_image_id(db, entry["sha256"])
|
|
||||||
if img_id is None:
|
|
||||||
unmatched.append({"kind": "tag", **entry})
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
tag_id = await _resolve_tag_id(
|
|
||||||
db, entry["tag_name"], entry.get("tag_kind") or "general",
|
|
||||||
)
|
|
||||||
if tag_id is None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
# Skip if association already exists.
|
|
||||||
already = (await db.execute(
|
|
||||||
select(image_tag.c.image_record_id).where(
|
|
||||||
image_tag.c.image_record_id == img_id,
|
|
||||||
image_tag.c.tag_id == tag_id,
|
|
||||||
)
|
|
||||||
)).first()
|
|
||||||
if already is not None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
if dry_run:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
continue
|
|
||||||
await db.execute(image_tag.insert().values(
|
|
||||||
image_record_id=img_id, tag_id=tag_id, source="manual",
|
|
||||||
))
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
|
|
||||||
# 3. Series pages.
|
|
||||||
for entry in manifest.get("series_pages", []):
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
img_id = await _sha_to_image_id(db, entry["sha256"])
|
|
||||||
if img_id is None:
|
|
||||||
unmatched.append({"kind": "series", **entry})
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
series_tag_id = await _resolve_tag_id(db, entry["series_tag_name"], "series")
|
|
||||||
if series_tag_id is None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(SeriesPage).where(SeriesPage.image_id == img_id)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
if dry_run:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
continue
|
|
||||||
db.add(SeriesPage(
|
|
||||||
series_tag_id=series_tag_id,
|
|
||||||
image_id=img_id,
|
|
||||||
page_number=entry["page_number"],
|
|
||||||
))
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
|
|
||||||
# 4. Image posts (schema v2) → Source + Post + ImageProvenance.
|
|
||||||
# Restores IR PostMetadata as FC's downloader-track provenance,
|
|
||||||
# so the modal's ProvenancePanel surfaces title/description/
|
|
||||||
# source URL/publish date the same way it does for live
|
|
||||||
# gallery-dl downloads.
|
|
||||||
for entry in manifest.get("image_posts", []):
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
platform = entry.get("platform")
|
|
||||||
artist_name = entry.get("artist")
|
|
||||||
if not platform or not artist_name:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
aid = await _ensure_artist_id(db, artist_name, dry_run)
|
|
||||||
if aid is None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
url = _profile_url(platform, slugify(artist_name))
|
|
||||||
if url is None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
source_id = await _find_or_create_source(
|
|
||||||
db, artist_id=aid, platform=platform, url=url, dry_run=dry_run,
|
|
||||||
)
|
|
||||||
if source_id is None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
post_id = await _find_or_create_post(
|
|
||||||
db, source_id=source_id,
|
|
||||||
external_post_id=entry.get("post_id") or "",
|
|
||||||
title=entry.get("title"),
|
|
||||||
description=entry.get("description"),
|
|
||||||
post_url=entry.get("source_url"),
|
|
||||||
post_date_iso=entry.get("published_at"),
|
|
||||||
attachment_count=entry.get("attachment_count") or 0,
|
|
||||||
dry_run=dry_run,
|
|
||||||
)
|
|
||||||
if post_id is None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
for sha in entry.get("image_sha256s", []):
|
|
||||||
img_id = await _sha_to_image_id(db, sha)
|
|
||||||
if img_id is None:
|
|
||||||
unmatched.append({
|
|
||||||
"kind": "post", "sha256": sha,
|
|
||||||
"post_id": entry.get("post_id"),
|
|
||||||
})
|
|
||||||
continue
|
|
||||||
inserted = await _ensure_provenance(
|
|
||||||
db, image_id=img_id, post_id=post_id,
|
|
||||||
source_id=source_id, dry_run=dry_run,
|
|
||||||
)
|
|
||||||
if inserted:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
else:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
|
|
||||||
if not dry_run:
|
|
||||||
await db.commit()
|
|
||||||
return {"counts": counts, "unmatched": unmatched}
|
|
||||||
@@ -1,80 +0,0 @@
|
|||||||
"""Post-migration verification: row counts + sha256 sampling."""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
from ...models import Artist, Credential, ImageRecord, Source, Tag
|
|
||||||
|
|
||||||
|
|
||||||
async def verify_async(db: AsyncSession, *, expected: dict | None = None) -> dict:
|
|
||||||
"""Return per-check status dicts. `expected` is optional row-count
|
|
||||||
assertions; checks default to status='ok' when no expected provided."""
|
|
||||||
expected = expected or {}
|
|
||||||
results: dict[str, dict] = {}
|
|
||||||
|
|
||||||
checks = {
|
|
||||||
"artist_subscriptions": (
|
|
||||||
select(func.count(Artist.id)).where(Artist.is_subscription.is_(True))
|
|
||||||
),
|
|
||||||
"source_count": select(func.count(Source.id)),
|
|
||||||
"credential_count": select(func.count(Credential.id)),
|
|
||||||
"tag_count": select(func.count(Tag.id)),
|
|
||||||
"image_record_imported_or_downloaded": (
|
|
||||||
select(func.count(ImageRecord.id))
|
|
||||||
.where(ImageRecord.origin.in_(["imported_filesystem", "downloaded"]))
|
|
||||||
),
|
|
||||||
}
|
|
||||||
for name, stmt in checks.items():
|
|
||||||
actual = (await db.execute(stmt)).scalar_one()
|
|
||||||
exp = expected.get(name)
|
|
||||||
status = "ok" if exp is None or exp == actual else "mismatch"
|
|
||||||
results[name] = {"status": status, "actual": int(actual), "expected": exp}
|
|
||||||
return results
|
|
||||||
|
|
||||||
|
|
||||||
async def verify_sha256_sample(
|
|
||||||
db: AsyncSession, *, sample_size: int = 20,
|
|
||||||
) -> dict:
|
|
||||||
"""Sample N image_records; verify file exists + sha256 matches."""
|
|
||||||
rows = (await db.execute(
|
|
||||||
select(ImageRecord.id, ImageRecord.path, ImageRecord.sha256)
|
|
||||||
.order_by(func.random()).limit(sample_size)
|
|
||||||
)).all()
|
|
||||||
|
|
||||||
matched = 0
|
|
||||||
mismatched = 0
|
|
||||||
missing = 0
|
|
||||||
samples: list[dict] = []
|
|
||||||
for img_id, path, expected_sha in rows:
|
|
||||||
p = Path(path)
|
|
||||||
# Sync stdlib filesystem ops are intentional: this verify pass runs
|
|
||||||
# inside a Celery task under asyncio.run; no other awaitables compete
|
|
||||||
# for the loop. Same pattern as download_service.py.
|
|
||||||
if not p.exists(): # noqa: ASYNC240
|
|
||||||
missing += 1
|
|
||||||
samples.append({"id": img_id, "path": path, "result": "missing"})
|
|
||||||
continue
|
|
||||||
h = hashlib.sha256()
|
|
||||||
with p.open("rb") as f: # noqa: ASYNC230
|
|
||||||
for chunk in iter(lambda: f.read(65536), b""):
|
|
||||||
h.update(chunk)
|
|
||||||
if h.hexdigest() == expected_sha:
|
|
||||||
matched += 1
|
|
||||||
samples.append({"id": img_id, "result": "ok"})
|
|
||||||
else:
|
|
||||||
mismatched += 1
|
|
||||||
samples.append({
|
|
||||||
"id": img_id, "path": path, "result": "mismatch",
|
|
||||||
"expected_sha": expected_sha, "actual_sha": h.hexdigest(),
|
|
||||||
})
|
|
||||||
return {
|
|
||||||
"sample_size": len(rows),
|
|
||||||
"matched": matched,
|
|
||||||
"mismatched": mismatched,
|
|
||||||
"missing": missing,
|
|
||||||
"samples": samples,
|
|
||||||
}
|
|
||||||
@@ -56,9 +56,17 @@ class PostFeedService:
|
|||||||
artist_id: int | None = None,
|
artist_id: int | None = None,
|
||||||
platform: str | None = None,
|
platform: str | None = None,
|
||||||
limit: int = 24,
|
limit: int = 24,
|
||||||
|
direction: str = "older",
|
||||||
) -> dict:
|
) -> dict:
|
||||||
|
"""Paginate the feed from `cursor`. direction='older' walks back in
|
||||||
|
time (default, infinite-scroll down); direction='newer' walks forward
|
||||||
|
(scroll up in an anchored view). Items are always returned in feed
|
||||||
|
(descending) order; `next_cursor` points to the far edge in the
|
||||||
|
requested direction (null when exhausted)."""
|
||||||
if limit < 1 or limit > 100:
|
if limit < 1 or limit > 100:
|
||||||
raise ValueError("limit must be between 1 and 100")
|
raise ValueError("limit must be between 1 and 100")
|
||||||
|
if direction not in ("older", "newer"):
|
||||||
|
raise ValueError("direction must be 'older' or 'newer'")
|
||||||
|
|
||||||
sort_key = _sort_key()
|
sort_key = _sort_key()
|
||||||
stmt = (
|
stmt = (
|
||||||
@@ -72,22 +80,37 @@ class PostFeedService:
|
|||||||
stmt = stmt.where(Source.platform == platform)
|
stmt = stmt.where(Source.platform == platform)
|
||||||
if cursor:
|
if cursor:
|
||||||
cur_ts, cur_id = decode_cursor(cursor)
|
cur_ts, cur_id = decode_cursor(cursor)
|
||||||
stmt = stmt.where(
|
if direction == "older":
|
||||||
or_(
|
stmt = stmt.where(or_(
|
||||||
sort_key < cur_ts,
|
sort_key < cur_ts,
|
||||||
and_(sort_key == cur_ts, Post.id < cur_id),
|
and_(sort_key == cur_ts, Post.id < cur_id),
|
||||||
)
|
))
|
||||||
)
|
else:
|
||||||
|
stmt = stmt.where(or_(
|
||||||
|
sort_key > cur_ts,
|
||||||
|
and_(sort_key == cur_ts, Post.id > cur_id),
|
||||||
|
))
|
||||||
|
|
||||||
stmt = stmt.order_by(sort_key.desc(), Post.id.desc()).limit(limit + 1)
|
if direction == "older":
|
||||||
|
stmt = stmt.order_by(sort_key.desc(), Post.id.desc())
|
||||||
|
else:
|
||||||
|
stmt = stmt.order_by(sort_key.asc(), Post.id.asc())
|
||||||
|
stmt = stmt.limit(limit + 1)
|
||||||
rows = (await self.session.execute(stmt)).all()
|
rows = (await self.session.execute(stmt)).all()
|
||||||
|
|
||||||
|
has_more = len(rows) > limit
|
||||||
|
rows = rows[:limit]
|
||||||
|
if direction == "newer":
|
||||||
|
# Fetched ascending (closest-newer first); flip to feed order.
|
||||||
|
rows = list(reversed(rows))
|
||||||
|
|
||||||
next_cursor: str | None = None
|
next_cursor: str | None = None
|
||||||
if len(rows) > limit:
|
if has_more and rows:
|
||||||
last_post, _, _ = rows[limit - 1]
|
# Far edge in the travel direction: oldest row going older,
|
||||||
last_key = last_post.post_date or last_post.downloaded_at
|
# newest row going newer (rows is descending for display).
|
||||||
next_cursor = encode_cursor(last_key, last_post.id)
|
edge_post = rows[-1][0] if direction == "older" else rows[0][0]
|
||||||
rows = rows[:limit]
|
edge_key = edge_post.post_date or edge_post.downloaded_at
|
||||||
|
next_cursor = encode_cursor(edge_key, edge_post.id)
|
||||||
|
|
||||||
post_ids = [p.id for p, _, _ in rows]
|
post_ids = [p.id for p, _, _ in rows]
|
||||||
thumbs_map = await self._thumbnails_for(post_ids)
|
thumbs_map = await self._thumbnails_for(post_ids)
|
||||||
@@ -99,6 +122,49 @@ class PostFeedService:
|
|||||||
]
|
]
|
||||||
return {"items": items, "next_cursor": next_cursor}
|
return {"items": items, "next_cursor": next_cursor}
|
||||||
|
|
||||||
|
async def around(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
post_id: int,
|
||||||
|
artist_id: int | None = None,
|
||||||
|
platform: str | None = None,
|
||||||
|
limit: int = 12,
|
||||||
|
) -> dict | None:
|
||||||
|
"""A window centered on `post_id`: up to `limit` newer posts + the
|
||||||
|
post + up to `limit` older posts, in feed (descending) order, with a
|
||||||
|
cursor for each end. Returns None if the post doesn't exist."""
|
||||||
|
anchor = (await self.session.execute(
|
||||||
|
select(Post, Artist, Source)
|
||||||
|
.join(Source, Post.source_id == Source.id)
|
||||||
|
.join(Artist, Source.artist_id == Artist.id)
|
||||||
|
.where(Post.id == post_id)
|
||||||
|
)).one_or_none()
|
||||||
|
if anchor is None:
|
||||||
|
return None
|
||||||
|
anchor_post, anchor_artist, anchor_source = anchor
|
||||||
|
anchor_key = anchor_post.post_date or anchor_post.downloaded_at
|
||||||
|
anchor_cursor = encode_cursor(anchor_key, anchor_post.id)
|
||||||
|
|
||||||
|
older = await self.scroll(
|
||||||
|
cursor=anchor_cursor, artist_id=artist_id, platform=platform,
|
||||||
|
limit=limit, direction="older",
|
||||||
|
)
|
||||||
|
newer = await self.scroll(
|
||||||
|
cursor=anchor_cursor, artist_id=artist_id, platform=platform,
|
||||||
|
limit=limit, direction="newer",
|
||||||
|
)
|
||||||
|
thumbs_map = await self._thumbnails_for([anchor_post.id])
|
||||||
|
atts_map = await self._attachments_for([anchor_post.id])
|
||||||
|
anchor_item = self._to_dict(
|
||||||
|
anchor_post, anchor_artist, anchor_source, thumbs_map, atts_map,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"items": newer["items"] + [anchor_item] + older["items"],
|
||||||
|
"cursor_older": older["next_cursor"],
|
||||||
|
"cursor_newer": newer["next_cursor"],
|
||||||
|
"anchor_id": anchor_post.id,
|
||||||
|
}
|
||||||
|
|
||||||
async def get_post(self, post_id: int) -> dict | None:
|
async def get_post(self, post_id: int) -> dict | None:
|
||||||
row = (await self.session.execute(
|
row = (await self.session.execute(
|
||||||
select(Post, Artist, Source)
|
select(Post, Artist, Source)
|
||||||
@@ -141,6 +207,7 @@ class PostFeedService:
|
|||||||
ImageRecord.primary_post_id,
|
ImageRecord.primary_post_id,
|
||||||
ImageRecord.sha256,
|
ImageRecord.sha256,
|
||||||
ImageRecord.mime,
|
ImageRecord.mime,
|
||||||
|
ImageRecord.thumbnail_path,
|
||||||
func.row_number().over(
|
func.row_number().over(
|
||||||
partition_by=ImageRecord.primary_post_id,
|
partition_by=ImageRecord.primary_post_id,
|
||||||
order_by=ImageRecord.id.asc(),
|
order_by=ImageRecord.id.asc(),
|
||||||
@@ -154,18 +221,18 @@ class PostFeedService:
|
|||||||
)
|
)
|
||||||
stmt = select(
|
stmt = select(
|
||||||
ranked.c.id, ranked.c.primary_post_id,
|
ranked.c.id, ranked.c.primary_post_id,
|
||||||
ranked.c.sha256, ranked.c.mime, ranked.c.total,
|
ranked.c.sha256, ranked.c.mime, ranked.c.thumbnail_path, ranked.c.total,
|
||||||
)
|
)
|
||||||
if limit is not None:
|
if limit is not None:
|
||||||
stmt = stmt.where(ranked.c.rn <= limit)
|
stmt = stmt.where(ranked.c.rn <= limit)
|
||||||
rows = (await self.session.execute(stmt)).all()
|
rows = (await self.session.execute(stmt)).all()
|
||||||
|
|
||||||
out: dict[int, dict] = {pid: {"thumbs": [], "more": 0} for pid in post_ids}
|
out: dict[int, dict] = {pid: {"thumbs": [], "more": 0} for pid in post_ids}
|
||||||
for img_id, pid, sha, mime, total in rows:
|
for img_id, pid, sha, mime, tp, total in rows:
|
||||||
entry = out.setdefault(pid, {"thumbs": [], "more": 0})
|
entry = out.setdefault(pid, {"thumbs": [], "more": 0})
|
||||||
entry["thumbs"].append({
|
entry["thumbs"].append({
|
||||||
"image_id": img_id,
|
"image_id": img_id,
|
||||||
"thumbnail_url": thumbnail_url(sha, mime),
|
"thumbnail_url": thumbnail_url(tp, sha, mime),
|
||||||
"mime": mime,
|
"mime": mime,
|
||||||
})
|
})
|
||||||
# `total` is constant per partition; overflow = total - THUMBNAIL_LIMIT.
|
# `total` is constant per partition; overflow = total - THUMBNAIL_LIMIT.
|
||||||
|
|||||||
@@ -0,0 +1,106 @@
|
|||||||
|
"""Layer-2 one-shot re-download remediation for corrupt imported files.
|
||||||
|
|
||||||
|
When an import fails on a file that came from a known, pollable
|
||||||
|
subscription Source, deleting the bad copy and re-running the source's
|
||||||
|
downloader can fetch a fresh, unblemished copy. This only helps when:
|
||||||
|
|
||||||
|
- the corruption is in transit / on disk (not at the source), AND
|
||||||
|
- the file resolves to an ENABLED Source with a real feed URL
|
||||||
|
(a `sidecar:<platform>:<slug>` synthetic anchor is not pollable),
|
||||||
|
AND
|
||||||
|
- we haven't already re-fetched this task once (bounded by
|
||||||
|
ImportTask.refetched so source-side corruption can't loop).
|
||||||
|
|
||||||
|
Filesystem-only imports with no resolvable Source return 'no_source' —
|
||||||
|
the operator's only remediation there is to replace the file on disk.
|
||||||
|
|
||||||
|
Operator-requested 2026-05-28 (Layer 2).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from ..models import Artist, ImportTask, Source
|
||||||
|
from ..utils.paths import derive_top_level_artist
|
||||||
|
from ..utils.sidecar import find_sidecar, parse_sidecar
|
||||||
|
from ..utils.slug import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_refetch_source(
|
||||||
|
session: Session, source_path: str, import_root: Path,
|
||||||
|
) -> Source | None:
|
||||||
|
"""Find an enabled, real-URL Source for the file's (artist, platform),
|
||||||
|
or None when nothing re-pollable resolves."""
|
||||||
|
path = Path(source_path)
|
||||||
|
sc = find_sidecar(path)
|
||||||
|
if sc is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
data = json.loads(sc.read_text("utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
return None
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return None
|
||||||
|
sd = parse_sidecar(data)
|
||||||
|
if not sd.platform:
|
||||||
|
return None
|
||||||
|
artist_name = derive_top_level_artist(path, import_root)
|
||||||
|
if not artist_name:
|
||||||
|
return None
|
||||||
|
artist = session.execute(
|
||||||
|
select(Artist).where(Artist.slug == slugify(artist_name))
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if artist is None:
|
||||||
|
return None
|
||||||
|
src = session.execute(
|
||||||
|
select(Source)
|
||||||
|
.where(
|
||||||
|
Source.artist_id == artist.id,
|
||||||
|
Source.platform == sd.platform,
|
||||||
|
Source.enabled.is_(True),
|
||||||
|
)
|
||||||
|
.order_by(Source.id.asc())
|
||||||
|
).scalars().first()
|
||||||
|
if src is None:
|
||||||
|
return None
|
||||||
|
if (src.url or "").startswith("sidecar:"):
|
||||||
|
return None # synthetic anchor — not a pollable feed
|
||||||
|
return src
|
||||||
|
|
||||||
|
|
||||||
|
def attempt_refetch(
|
||||||
|
session: Session, task: ImportTask, import_root: Path,
|
||||||
|
) -> dict:
|
||||||
|
"""Delete the corrupt file, mark the task refetched, and trigger ONE
|
||||||
|
source re-check. Idempotent/bounded: a task already refetched (or
|
||||||
|
with no resolvable Source) is a no-op. Commits."""
|
||||||
|
if task.refetched:
|
||||||
|
return {"status": "already_refetched"}
|
||||||
|
src = resolve_refetch_source(session, task.source_path, import_root)
|
||||||
|
if src is None:
|
||||||
|
return {"status": "no_source"}
|
||||||
|
|
||||||
|
# Remove the bad copy so gallery-dl (skip_existing) re-fetches it on
|
||||||
|
# the source re-check instead of skipping the still-present corrupt
|
||||||
|
# file.
|
||||||
|
try:
|
||||||
|
Path(task.source_path).unlink(missing_ok=True)
|
||||||
|
except OSError as exc:
|
||||||
|
log.warning("refetch unlink failed for %s: %s", task.source_path, exc)
|
||||||
|
|
||||||
|
task.refetched = True
|
||||||
|
session.add(task)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
# Lazy import to avoid a tasks→services→tasks import cycle at module
|
||||||
|
# load. download_source.delay() is sync-safe in any context.
|
||||||
|
from ..tasks.download import download_source
|
||||||
|
|
||||||
|
download_source.delay(src.id)
|
||||||
|
return {"status": "refetch_queued", "source_id": src.id}
|
||||||
@@ -9,15 +9,33 @@ from __future__ import annotations
|
|||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
from sqlalchemy.orm import selectinload
|
from sqlalchemy.orm import selectinload
|
||||||
|
|
||||||
from ..models import Artist, ImportSettings, Source
|
from ..models import AppSetting, Artist, ImportSettings, Source
|
||||||
|
|
||||||
MIN_INTERVAL_SECONDS = 60
|
MIN_INTERVAL_SECONDS = 60
|
||||||
MAX_INTERVAL_SECONDS = 86400
|
MAX_INTERVAL_SECONDS = 86400
|
||||||
MAX_BACKOFF_EXPONENT = 6
|
MAX_BACKOFF_EXPONENT = 6
|
||||||
|
|
||||||
|
# AppSetting key stamped every time the Beat tick fires (see scan.py). The
|
||||||
|
# tick runs every 60s; the UI flags the scheduler as stalled if the last
|
||||||
|
# stamp is older than a few minutes.
|
||||||
|
SCHEDULER_LAST_TICK_KEY = "scheduler_last_tick_at"
|
||||||
|
|
||||||
|
# AppSetting key prefix for per-platform rate-limit cooldowns. When a
|
||||||
|
# download surfaces ErrorType.RATE_LIMITED, every other source on the same
|
||||||
|
# platform is deferred for PLATFORM_RATE_LIMIT_COOLDOWN_SECONDS so the next
|
||||||
|
# scan tick doesn't fire a burst of due same-platform sources back into the
|
||||||
|
# same limit. Per-source consecutive_failures backoff still applies on top
|
||||||
|
# of this — but this is PREVENTIVE (kills the same-tick burst from N due
|
||||||
|
# sources hammering the platform at once), while consecutive_failures is
|
||||||
|
# REACTIVE (slows the offender down over many cycles). Operator-confirmed
|
||||||
|
# 2026-05-30.
|
||||||
|
PLATFORM_COOLDOWN_KEY_PREFIX = "platform_cooldown:"
|
||||||
|
PLATFORM_RATE_LIMIT_COOLDOWN_SECONDS = 900 # 15 min
|
||||||
|
|
||||||
|
|
||||||
def compute_effective_interval(
|
def compute_effective_interval(
|
||||||
source: Source, artist: Artist, settings: ImportSettings,
|
source: Source, artist: Artist, settings: ImportSettings,
|
||||||
@@ -40,10 +58,76 @@ def compute_effective_interval(
|
|||||||
return max(MIN_INTERVAL_SECONDS, min(MAX_INTERVAL_SECONDS, raw))
|
return max(MIN_INTERVAL_SECONDS, min(MAX_INTERVAL_SECONDS, raw))
|
||||||
|
|
||||||
|
|
||||||
|
async def set_platform_cooldown(
|
||||||
|
session: AsyncSession, platform: str,
|
||||||
|
seconds: int = PLATFORM_RATE_LIMIT_COOLDOWN_SECONDS,
|
||||||
|
) -> None:
|
||||||
|
"""Stamp a cooldown expiry on the given platform so select_due_sources
|
||||||
|
skips every source on that platform until it expires.
|
||||||
|
|
||||||
|
Called when a download surfaces ErrorType.RATE_LIMITED so the other
|
||||||
|
sources on the same platform don't all retry into the same rate limit.
|
||||||
|
Caller is responsible for committing the session.
|
||||||
|
|
||||||
|
Uses INSERT...ON CONFLICT DO UPDATE so two concurrent workers hitting
|
||||||
|
the same platform's rate limit don't race: a SELECT-then-INSERT pattern
|
||||||
|
would let the loser's whole transaction (including the source-health
|
||||||
|
update + event finalize) roll back on a unique-violation, stranding
|
||||||
|
that event. Atomic upsert avoids that.
|
||||||
|
"""
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
expires_at = (now + timedelta(seconds=seconds)).isoformat()
|
||||||
|
key = f"{PLATFORM_COOLDOWN_KEY_PREFIX}{platform}"
|
||||||
|
stmt = pg_insert(AppSetting.__table__).values(
|
||||||
|
key=key, value=expires_at, updated_at=now,
|
||||||
|
).on_conflict_do_update(
|
||||||
|
index_elements=["key"],
|
||||||
|
set_={"value": expires_at, "updated_at": now},
|
||||||
|
)
|
||||||
|
await session.execute(stmt)
|
||||||
|
|
||||||
|
|
||||||
|
async def active_platform_cooldowns(session: AsyncSession) -> dict[str, datetime]:
|
||||||
|
"""Return {platform: expires_at} for platforms whose cooldown is still
|
||||||
|
in the future. Expired rows are ignored (a future maintenance sweep can
|
||||||
|
delete them; they don't affect routing decisions on their own).
|
||||||
|
|
||||||
|
Exposed beyond scheduler_service so the manual check endpoint
|
||||||
|
(`/api/sources/<id>/check`) can defer bulk retries that would bowl
|
||||||
|
into the same rate limit the cooldown is preventing.
|
||||||
|
"""
|
||||||
|
rows = (await session.execute(
|
||||||
|
select(AppSetting.key, AppSetting.value)
|
||||||
|
.where(AppSetting.key.startswith(PLATFORM_COOLDOWN_KEY_PREFIX))
|
||||||
|
)).all()
|
||||||
|
if not rows:
|
||||||
|
return {}
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
active: dict[str, datetime] = {}
|
||||||
|
for key, value in rows:
|
||||||
|
try:
|
||||||
|
expires_at = datetime.fromisoformat(value)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
continue
|
||||||
|
if expires_at > now:
|
||||||
|
active[key[len(PLATFORM_COOLDOWN_KEY_PREFIX):]] = expires_at
|
||||||
|
return active
|
||||||
|
|
||||||
|
|
||||||
async def select_due_sources(session: AsyncSession) -> list[Source]:
|
async def select_due_sources(session: AsyncSession) -> list[Source]:
|
||||||
"""Sources where (enabled, artist.auto_check) and now >= last_checked_at + effective_interval.
|
"""Sources where (enabled, artist.auto_check) and now >= last_checked_at + effective_interval.
|
||||||
|
|
||||||
Never-checked sources (last_checked_at IS NULL) are always due.
|
Never-checked sources (last_checked_at IS NULL) are always due. Sources
|
||||||
|
whose platform is currently in a rate-limit cooldown are excluded — the
|
||||||
|
cooldown is the preventive half of the burst-prevention pair (per-source
|
||||||
|
consecutive_failures backoff handles the offending source itself).
|
||||||
|
|
||||||
|
Ordering: last_checked_at ASC NULLS FIRST, then id. Never-checked
|
||||||
|
sources go first, then the longest-since-checked, so the most overdue
|
||||||
|
sources hit Celery's FIFO download queue first. Anti-starvation: if
|
||||||
|
queue throughput ever falls below the tick rate, a freshly-rerun source
|
||||||
|
can't keep cutting in line ahead of one that hasn't been checked at all.
|
||||||
|
Operator-confirmed 2026-05-30.
|
||||||
"""
|
"""
|
||||||
rows = (await session.execute(
|
rows = (await session.execute(
|
||||||
select(Source)
|
select(Source)
|
||||||
@@ -51,15 +135,17 @@ async def select_due_sources(session: AsyncSession) -> list[Source]:
|
|||||||
.join(Artist, Source.artist_id == Artist.id)
|
.join(Artist, Source.artist_id == Artist.id)
|
||||||
.where(Source.enabled.is_(True))
|
.where(Source.enabled.is_(True))
|
||||||
.where(Artist.auto_check.is_(True))
|
.where(Artist.auto_check.is_(True))
|
||||||
|
.order_by(Source.last_checked_at.asc().nulls_first(), Source.id)
|
||||||
)).scalars().all()
|
)).scalars().all()
|
||||||
|
|
||||||
settings = (await session.execute(
|
cooldowns = await active_platform_cooldowns(session)
|
||||||
select(ImportSettings).where(ImportSettings.id == 1)
|
settings = await ImportSettings.load(session)
|
||||||
)).scalar_one()
|
|
||||||
|
|
||||||
now = datetime.now(UTC)
|
now = datetime.now(UTC)
|
||||||
due: list[Source] = []
|
due: list[Source] = []
|
||||||
for s in rows:
|
for s in rows:
|
||||||
|
if s.platform in cooldowns:
|
||||||
|
continue
|
||||||
interval = compute_effective_interval(s, s.artist, settings)
|
interval = compute_effective_interval(s, s.artist, settings)
|
||||||
if s.last_checked_at is None:
|
if s.last_checked_at is None:
|
||||||
due.append(s)
|
due.append(s)
|
||||||
@@ -78,3 +164,65 @@ def compute_next_check_at(
|
|||||||
return None
|
return None
|
||||||
interval = compute_effective_interval(source, artist, settings)
|
interval = compute_effective_interval(source, artist, settings)
|
||||||
return source.last_checked_at + timedelta(seconds=interval)
|
return source.last_checked_at + timedelta(seconds=interval)
|
||||||
|
|
||||||
|
|
||||||
|
async def record_tick(session: AsyncSession) -> None:
|
||||||
|
"""Stamp the current time on the SCHEDULER_LAST_TICK_KEY AppSetting.
|
||||||
|
|
||||||
|
Called once per Beat tick so the UI can prove the scheduler is alive.
|
||||||
|
Commits its own write so the stamp survives even if the rest of the
|
||||||
|
tick errors out.
|
||||||
|
"""
|
||||||
|
now_iso = datetime.now(UTC).isoformat()
|
||||||
|
row = (await session.execute(
|
||||||
|
select(AppSetting).where(AppSetting.key == SCHEDULER_LAST_TICK_KEY)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if row is None:
|
||||||
|
session.add(AppSetting(key=SCHEDULER_LAST_TICK_KEY, value=now_iso))
|
||||||
|
else:
|
||||||
|
row.value = now_iso
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
async def scheduler_status(session: AsyncSession) -> dict:
|
||||||
|
"""Summarise scheduler health for the dashboard.
|
||||||
|
|
||||||
|
Returns last_tick_at (when Beat last fired), next_due_at (earliest
|
||||||
|
upcoming scheduled check across enabled auto-check sources), due_now
|
||||||
|
(how many are due right now), and auto_sources (total under schedule).
|
||||||
|
"""
|
||||||
|
last_tick_at = (await session.execute(
|
||||||
|
select(AppSetting.value).where(AppSetting.key == SCHEDULER_LAST_TICK_KEY)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
|
||||||
|
rows = (await session.execute(
|
||||||
|
select(Source)
|
||||||
|
.options(selectinload(Source.artist))
|
||||||
|
.join(Artist, Source.artist_id == Artist.id)
|
||||||
|
.where(Source.enabled.is_(True))
|
||||||
|
.where(Artist.auto_check.is_(True))
|
||||||
|
)).scalars().all()
|
||||||
|
settings = await ImportSettings.load(session)
|
||||||
|
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
due_now = 0
|
||||||
|
next_due_at: datetime | None = None
|
||||||
|
for s in rows:
|
||||||
|
if s.last_checked_at is None:
|
||||||
|
due_now += 1
|
||||||
|
continue
|
||||||
|
nca = compute_next_check_at(s, s.artist, settings)
|
||||||
|
if nca is None or nca <= now:
|
||||||
|
due_now += 1
|
||||||
|
elif next_due_at is None or nca < next_due_at:
|
||||||
|
next_due_at = nca
|
||||||
|
|
||||||
|
cooldowns = await active_platform_cooldowns(session)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"last_tick_at": last_tick_at,
|
||||||
|
"next_due_at": next_due_at.isoformat() if next_due_at else None,
|
||||||
|
"due_now": due_now,
|
||||||
|
"auto_sources": len(rows),
|
||||||
|
"platform_cooldowns": {p: dt.isoformat() for p, dt in cooldowns.items()},
|
||||||
|
}
|
||||||
|
|||||||
@@ -63,6 +63,7 @@ class SeriesService:
|
|||||||
ImageRecord.sha256,
|
ImageRecord.sha256,
|
||||||
ImageRecord.mime,
|
ImageRecord.mime,
|
||||||
ImageRecord.path,
|
ImageRecord.path,
|
||||||
|
ImageRecord.thumbnail_path,
|
||||||
)
|
)
|
||||||
.join(ImageRecord, ImageRecord.id == SeriesPage.image_id)
|
.join(ImageRecord, ImageRecord.id == SeriesPage.image_id)
|
||||||
.where(SeriesPage.series_tag_id == series_tag_id)
|
.where(SeriesPage.series_tag_id == series_tag_id)
|
||||||
@@ -75,7 +76,7 @@ class SeriesService:
|
|||||||
{
|
{
|
||||||
"image_id": r.image_id,
|
"image_id": r.image_id,
|
||||||
"page_number": r.page_number,
|
"page_number": r.page_number,
|
||||||
"thumbnail_url": thumbnail_url(r.sha256, r.mime),
|
"thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime),
|
||||||
"image_url": f"/images/{r.path.split('/images/', 1)[-1]}",
|
"image_url": f"/images/{r.path.split('/images/', 1)[-1]}",
|
||||||
}
|
}
|
||||||
for r in rows
|
for r in rows
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ class ShowcaseService:
|
|||||||
"mime": r.mime,
|
"mime": r.mime,
|
||||||
"width": r.width,
|
"width": r.width,
|
||||||
"height": r.height,
|
"height": r.height,
|
||||||
"thumbnail_url": thumbnail_url(r.sha256, r.mime),
|
"thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime),
|
||||||
}
|
}
|
||||||
for r in rows
|
for r in rows
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -120,9 +120,7 @@ class SourceService:
|
|||||||
return config
|
return config
|
||||||
|
|
||||||
async def _load_settings(self) -> ImportSettings:
|
async def _load_settings(self) -> ImportSettings:
|
||||||
return (await self.session.execute(
|
return await ImportSettings.load(self.session)
|
||||||
select(ImportSettings).where(ImportSettings.id == 1)
|
|
||||||
)).scalar_one()
|
|
||||||
|
|
||||||
def _build_record(
|
def _build_record(
|
||||||
self, source: Source, artist: Artist, settings: ImportSettings,
|
self, source: Source, artist: Artist, settings: ImportSettings,
|
||||||
@@ -151,14 +149,27 @@ class SourceService:
|
|||||||
settings = await self._load_settings()
|
settings = await self._load_settings()
|
||||||
return self._build_record(source, artist, settings)
|
return self._build_record(source, artist, settings)
|
||||||
|
|
||||||
async def list(self, artist_id: int | None = None) -> list[SourceRecord]:
|
async def list(
|
||||||
stmt = (
|
self, artist_id: int | None = None, failing: bool = False,
|
||||||
select(Source, Artist)
|
include_synthetic: bool = False,
|
||||||
.join(Artist, Artist.id == Source.artist_id)
|
) -> list[SourceRecord]:
|
||||||
.order_by(Artist.name.asc(), Source.id.asc())
|
stmt = select(Source, Artist).join(Artist, Artist.id == Source.artist_id)
|
||||||
)
|
|
||||||
if artist_id is not None:
|
if artist_id is not None:
|
||||||
stmt = stmt.where(Source.artist_id == artist_id)
|
stmt = stmt.where(Source.artist_id == artist_id)
|
||||||
|
if not include_synthetic:
|
||||||
|
# Filesystem-import sidecar anchors (importer._source_for_sidecar)
|
||||||
|
# have url='sidecar:<platform>:<slug>' and exist only to give
|
||||||
|
# imported Posts a NOT-NULL Source FK. They aren't pollable
|
||||||
|
# feeds; the Subscriptions UI used to render them as phantom
|
||||||
|
# subscriptions. Hide by default.
|
||||||
|
stmt = stmt.where(~Source.url.like("sidecar:%"))
|
||||||
|
if failing:
|
||||||
|
# Worst-first so the rollup card surfaces the loudest failures.
|
||||||
|
stmt = stmt.where(Source.consecutive_failures > 0).order_by(
|
||||||
|
Source.consecutive_failures.desc(), Artist.name.asc(),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
stmt = stmt.order_by(Artist.name.asc(), Source.id.asc())
|
||||||
rows = (await self.session.execute(stmt)).all()
|
rows = (await self.session.execute(stmt)).all()
|
||||||
settings = await self._load_settings()
|
settings = await self._load_settings()
|
||||||
return [self._build_record(s, a, settings) for s, a in rows]
|
return [self._build_record(s, a, settings) for s, a in rows]
|
||||||
|
|||||||
@@ -115,12 +115,17 @@ class TagDirectoryService:
|
|||||||
.subquery()
|
.subquery()
|
||||||
)
|
)
|
||||||
stmt = (
|
stmt = (
|
||||||
select(sub.c.tag_id, ImageRecord.sha256, ImageRecord.mime)
|
select(
|
||||||
|
sub.c.tag_id,
|
||||||
|
ImageRecord.sha256,
|
||||||
|
ImageRecord.mime,
|
||||||
|
ImageRecord.thumbnail_path,
|
||||||
|
)
|
||||||
.join(ImageRecord, ImageRecord.id == sub.c.image_record_id)
|
.join(ImageRecord, ImageRecord.id == sub.c.image_record_id)
|
||||||
.where(sub.c.rn <= 3)
|
.where(sub.c.rn <= 3)
|
||||||
.order_by(sub.c.tag_id, sub.c.rn)
|
.order_by(sub.c.tag_id, sub.c.rn)
|
||||||
)
|
)
|
||||||
out: dict[int, list[str]] = {}
|
out: dict[int, list[str]] = {}
|
||||||
for tag_id, sha, mime in (await self.session.execute(stmt)).all():
|
for tag_id, sha, mime, tp in (await self.session.execute(stmt)).all():
|
||||||
out.setdefault(tag_id, []).append(thumbnail_url(sha, mime))
|
out.setdefault(tag_id, []).append(thumbnail_url(tp, sha, mime))
|
||||||
return out
|
return out
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
"""Per-invocation async session factory for Celery task modules.
|
||||||
|
|
||||||
|
Async engine connections are bound to the event loop. Each Celery task
|
||||||
|
runs its async body under a fresh ``asyncio.run()`` loop, so it needs its
|
||||||
|
own engine created (and disposed) within that loop — a process-wide async
|
||||||
|
engine would reuse loop-bound connections across tasks and raise "attached
|
||||||
|
to a different loop". So unlike the process-wide sync engine in
|
||||||
|
``_sync_engine.py``, this returns a fresh engine per call; the caller
|
||||||
|
disposes it (``await engine.dispose()``) when its loop ends.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||||
|
|
||||||
|
from ..config import get_config
|
||||||
|
|
||||||
|
|
||||||
|
def async_session_factory():
|
||||||
|
"""Return ``(sessionmaker, engine)`` bound to a fresh async engine."""
|
||||||
|
cfg = get_config()
|
||||||
|
engine = create_async_engine(cfg.database_url, future=True, pool_pre_ping=True)
|
||||||
|
return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False), engine
|
||||||
@@ -246,9 +246,7 @@ def prune_backups() -> dict:
|
|||||||
SessionLocal = _sync_session_factory()
|
SessionLocal = _sync_session_factory()
|
||||||
counts = {"db_deleted": 0, "images_deleted": 0, "files_unlinked": 0}
|
counts = {"db_deleted": 0, "images_deleted": 0, "files_unlinked": 0}
|
||||||
with SessionLocal() as session:
|
with SessionLocal() as session:
|
||||||
s = session.execute(
|
s = ImportSettings.load_sync(session)
|
||||||
select(ImportSettings).where(ImportSettings.id == 1)
|
|
||||||
).scalar_one()
|
|
||||||
for kind, keep in (
|
for kind, keep in (
|
||||||
("db", s.backup_db_keep_last_n),
|
("db", s.backup_db_keep_last_n),
|
||||||
("images", s.backup_images_keep_last_n),
|
("images", s.backup_images_keep_last_n),
|
||||||
@@ -286,9 +284,7 @@ def backup_db_nightly() -> dict:
|
|||||||
either {'skipped': '<reason>'} or {'dispatched': '<task_id>'}."""
|
either {'skipped': '<reason>'} or {'dispatched': '<task_id>'}."""
|
||||||
SessionLocal = _sync_session_factory()
|
SessionLocal = _sync_session_factory()
|
||||||
with SessionLocal() as session:
|
with SessionLocal() as session:
|
||||||
s = session.execute(
|
s = ImportSettings.load_sync(session)
|
||||||
select(ImportSettings).where(ImportSettings.id == 1)
|
|
||||||
).scalar_one()
|
|
||||||
nightly_enabled = s.backup_db_nightly_enabled
|
nightly_enabled = s.backup_db_nightly_enabled
|
||||||
configured_hour = s.backup_db_nightly_hour_utc
|
configured_hour = s.backup_db_nightly_hour_utc
|
||||||
if not nightly_enabled:
|
if not nightly_enabled:
|
||||||
|
|||||||
@@ -3,12 +3,9 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.exc import DBAPIError, OperationalError
|
from sqlalchemy.exc import DBAPIError, OperationalError
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
||||||
|
|
||||||
from ..celery_app import celery
|
from ..celery_app import celery
|
||||||
from ..config import get_config
|
|
||||||
from ..models import ImportSettings
|
from ..models import ImportSettings
|
||||||
from ..services.credential_crypto import CredentialCrypto
|
from ..services.credential_crypto import CredentialCrypto
|
||||||
from ..services.credential_service import CredentialService
|
from ..services.credential_service import CredentialService
|
||||||
@@ -16,18 +13,13 @@ from ..services.download_service import DownloadService
|
|||||||
from ..services.gallery_dl import GalleryDLService
|
from ..services.gallery_dl import GalleryDLService
|
||||||
from ..services.importer import Importer
|
from ..services.importer import Importer
|
||||||
from ..services.thumbnailer import Thumbnailer
|
from ..services.thumbnailer import Thumbnailer
|
||||||
|
from ._async_session import async_session_factory
|
||||||
from .import_file import _sync_session_factory
|
from .import_file import _sync_session_factory
|
||||||
|
|
||||||
IMAGES_ROOT = Path("/images")
|
IMAGES_ROOT = Path("/images")
|
||||||
_KEY_PATH = IMAGES_ROOT / "secrets" / "credential_key.b64"
|
_KEY_PATH = IMAGES_ROOT / "secrets" / "credential_key.b64"
|
||||||
|
|
||||||
|
|
||||||
def _async_session_factory():
|
|
||||||
cfg = get_config()
|
|
||||||
engine = create_async_engine(cfg.database_url, future=True, pool_pre_ping=True)
|
|
||||||
return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False), engine
|
|
||||||
|
|
||||||
|
|
||||||
@celery.task(
|
@celery.task(
|
||||||
name="backend.app.tasks.download.download_source",
|
name="backend.app.tasks.download.download_source",
|
||||||
bind=True,
|
bind=True,
|
||||||
@@ -44,13 +36,11 @@ def download_source(self, source_id: int) -> int:
|
|||||||
"""Returns the DownloadEvent.id."""
|
"""Returns the DownloadEvent.id."""
|
||||||
|
|
||||||
async def _run():
|
async def _run():
|
||||||
async_factory, async_engine = _async_session_factory()
|
async_factory, async_engine = async_session_factory()
|
||||||
SyncFactory = _sync_session_factory()
|
SyncFactory = _sync_session_factory()
|
||||||
try:
|
try:
|
||||||
with SyncFactory() as sync_session:
|
with SyncFactory() as sync_session:
|
||||||
settings = sync_session.execute(
|
settings = ImportSettings.load_sync(sync_session)
|
||||||
select(ImportSettings).where(ImportSettings.id == 1)
|
|
||||||
).scalar_one()
|
|
||||||
rate_limit = settings.download_rate_limit_seconds
|
rate_limit = settings.download_rate_limit_seconds
|
||||||
validate_files = settings.download_validate_files
|
validate_files = settings.download_validate_files
|
||||||
|
|
||||||
@@ -64,9 +54,7 @@ def download_source(self, source_id: int) -> int:
|
|||||||
async with async_factory() as async_session:
|
async with async_factory() as async_session:
|
||||||
cred_service = CredentialService(async_session, crypto)
|
cred_service = CredentialService(async_session, crypto)
|
||||||
with SyncFactory() as sync_session:
|
with SyncFactory() as sync_session:
|
||||||
sync_settings = sync_session.execute(
|
sync_settings = ImportSettings.load_sync(sync_session)
|
||||||
select(ImportSettings).where(ImportSettings.id == 1)
|
|
||||||
).scalar_one()
|
|
||||||
importer = Importer(
|
importer = Importer(
|
||||||
session=sync_session,
|
session=sync_session,
|
||||||
images_root=IMAGES_ROOT,
|
images_root=IMAGES_ROOT,
|
||||||
|
|||||||
@@ -64,30 +64,13 @@ def _mark_failed(session, task, error_msg: str) -> None:
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
@celery.task(
|
def _run_import_task(import_task_id: int) -> dict:
|
||||||
name="backend.app.tasks.import_file.import_media_file",
|
"""Shared body for import_media_file + import_archive_file. The two
|
||||||
bind=True,
|
tasks differ ONLY in their Celery time limits (a single media file
|
||||||
autoretry_for=(OperationalError, DBAPIError, OSError),
|
is sub-second; an archive runs the full per-member pipeline inline
|
||||||
retry_backoff=5,
|
for every member and can take many minutes). Both flip the row to
|
||||||
retry_backoff_max=60,
|
'processing', dispatch to `_do_import`, and honor the
|
||||||
retry_jitter=True,
|
flip-to-terminal resilience contract.
|
||||||
max_retries=3,
|
|
||||||
soft_time_limit=300,
|
|
||||||
time_limit=360,
|
|
||||||
)
|
|
||||||
def import_media_file(self, import_task_id: int) -> dict:
|
|
||||||
"""Returns a dict so the eager-mode tests can assert without DB.
|
|
||||||
|
|
||||||
Decorator notes:
|
|
||||||
- autoretry_for: transient DB / filesystem errors retry with
|
|
||||||
exponential backoff (5s base, jitter, max 3 attempts). On final
|
|
||||||
give-up the task raises and acks_late=True (set globally on the
|
|
||||||
Celery app) does NOT redeliver — the recovery sweep catches the
|
|
||||||
row instead.
|
|
||||||
- soft_time_limit (300s) raises SoftTimeLimitExceeded in this
|
|
||||||
process so the task can mark its row failed before being killed.
|
|
||||||
- time_limit (360s) is the hard cap; SIGKILL if the soft signal
|
|
||||||
was swallowed.
|
|
||||||
"""
|
"""
|
||||||
SessionLocal = _sync_session_factory()
|
SessionLocal = _sync_session_factory()
|
||||||
with SessionLocal() as session:
|
with SessionLocal() as session:
|
||||||
@@ -103,24 +86,88 @@ def import_media_file(self, import_task_id: int) -> dict:
|
|||||||
try:
|
try:
|
||||||
return _do_import(session, task, import_task_id)
|
return _do_import(session, task, import_task_id)
|
||||||
except SoftTimeLimitExceeded:
|
except SoftTimeLimitExceeded:
|
||||||
_mark_failed(session, task, "soft_time_limit exceeded (>300s)")
|
_mark_failed(session, task, "soft_time_limit exceeded")
|
||||||
raise
|
raise
|
||||||
except (OperationalError, DBAPIError, OSError):
|
except (OperationalError, DBAPIError, OSError):
|
||||||
# Retryable per the decorator; do NOT mark failed (let
|
# Retryable per the decorator; do NOT mark failed (let
|
||||||
# autoretry have a clean go at it). If autoretry exhausts,
|
# autoretry have a clean go at it). If autoretry exhausts,
|
||||||
# the row stays 'processing' and the maintenance sweep
|
# the row stays 'processing' and the maintenance sweep
|
||||||
# flips it within 5 min.
|
# flips it.
|
||||||
raise
|
raise
|
||||||
except Exception as exc: # noqa: BLE001 — pipeline crash, mark + re-raise
|
except Exception as exc: # noqa: BLE001 — pipeline crash, mark + re-raise
|
||||||
_mark_failed(session, task, f"{type(exc).__name__}: {exc}")
|
_mark_failed(session, task, f"{type(exc).__name__}: {exc}")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.import_file.import_media_file",
|
||||||
|
bind=True,
|
||||||
|
autoretry_for=(OperationalError, DBAPIError, OSError),
|
||||||
|
retry_backoff=5,
|
||||||
|
retry_backoff_max=60,
|
||||||
|
retry_jitter=True,
|
||||||
|
max_retries=3,
|
||||||
|
soft_time_limit=300,
|
||||||
|
time_limit=360,
|
||||||
|
)
|
||||||
|
def import_media_file(self, import_task_id: int) -> dict:
|
||||||
|
"""Import ONE media file (or non-media → PostAttachment). Sub-second
|
||||||
|
for the common case; the tight 5-min soft limit keeps a genuinely
|
||||||
|
stuck single-file import detectable fast.
|
||||||
|
|
||||||
|
Decorator notes:
|
||||||
|
- autoretry_for: transient DB / filesystem errors retry with
|
||||||
|
exponential backoff (5s base, jitter, max 3 attempts). On final
|
||||||
|
give-up the task raises and acks_late=True (set globally on the
|
||||||
|
Celery app) does NOT redeliver — the recovery sweep catches the
|
||||||
|
row instead.
|
||||||
|
- soft_time_limit (300s) raises SoftTimeLimitExceeded in-process
|
||||||
|
so the task can mark its row failed before being killed.
|
||||||
|
- time_limit (360s) is the hard SIGKILL cap.
|
||||||
|
"""
|
||||||
|
return _run_import_task(import_task_id)
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.import_file.import_archive_file",
|
||||||
|
bind=True,
|
||||||
|
autoretry_for=(OperationalError, DBAPIError, OSError),
|
||||||
|
retry_backoff=5,
|
||||||
|
retry_backoff_max=60,
|
||||||
|
retry_jitter=True,
|
||||||
|
max_retries=3,
|
||||||
|
# Archives run the full per-member pipeline (sha256 + pHash + dedup
|
||||||
|
# query + copy + provenance) for EVERY media member inline, under a
|
||||||
|
# single task budget. A multi-hundred-member archive blows the
|
||||||
|
# 5-min media limit. soft=30min / hard=35min sizes for a large
|
||||||
|
# archive. Operator-flagged 2026-05-28 (target 1645019 hit the old
|
||||||
|
# shared 300s soft limit). The recovery sweep gives this task its
|
||||||
|
# own 40-min threshold via maintenance.TASK_STUCK_THRESHOLD_MINUTES
|
||||||
|
# so it isn't preempted while legitimately grinding through members.
|
||||||
|
soft_time_limit=1800,
|
||||||
|
time_limit=2100,
|
||||||
|
)
|
||||||
|
def import_archive_file(self, import_task_id: int) -> dict:
|
||||||
|
"""Import an archive: extract + run the per-member media pipeline for
|
||||||
|
every member inline, then preserve the archive as a PostAttachment.
|
||||||
|
Same body as import_media_file (dispatch is by file kind inside
|
||||||
|
Importer.import_one); split out purely for the larger time budget."""
|
||||||
|
return _run_import_task(import_task_id)
|
||||||
|
|
||||||
|
|
||||||
|
def enqueue_import(task_id: int, task_type: str) -> None:
|
||||||
|
"""Route an ImportTask to the right Celery task by its task_type.
|
||||||
|
Single source of truth for the media-vs-archive dispatch so the
|
||||||
|
scan, retry, and recovery-requeue paths stay in sync."""
|
||||||
|
if task_type == "archive":
|
||||||
|
import_archive_file.delay(task_id)
|
||||||
|
else:
|
||||||
|
import_media_file.delay(task_id)
|
||||||
|
|
||||||
|
|
||||||
def _do_import(session, task, import_task_id: int) -> dict:
|
def _do_import(session, task, import_task_id: int) -> dict:
|
||||||
"""Actual work, called from inside the resilience wrapper."""
|
"""Actual work, called from inside the resilience wrapper."""
|
||||||
settings = session.execute(
|
settings = ImportSettings.load_sync(session)
|
||||||
select(ImportSettings).where(ImportSettings.id == 1)
|
|
||||||
).scalar_one()
|
|
||||||
import_root = Path(settings.import_scan_path)
|
import_root = Path(settings.import_scan_path)
|
||||||
batch = session.get(ImportBatch, task.batch_id)
|
batch = session.get(ImportBatch, task.batch_id)
|
||||||
deep = bool(batch and batch.scan_mode == "deep")
|
deep = bool(batch and batch.scan_mode == "deep")
|
||||||
|
|||||||
@@ -1,22 +1,53 @@
|
|||||||
"""Periodic maintenance: recover stuck import tasks, garbage-collect old finished tasks."""
|
"""Periodic maintenance: recover stuck import tasks, garbage-collect old finished tasks."""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from sqlalchemy import delete, select, update
|
from sqlalchemy import and_, delete, or_, select, update
|
||||||
|
|
||||||
from ..celery_app import celery
|
from ..celery_app import celery
|
||||||
from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask, TaskRun
|
from ..models import (
|
||||||
|
DownloadEvent,
|
||||||
|
ImageRecord,
|
||||||
|
ImportSettings,
|
||||||
|
ImportTask,
|
||||||
|
Source,
|
||||||
|
TaskRun,
|
||||||
|
)
|
||||||
from ..utils.phash import compute_phash
|
from ..utils.phash import compute_phash
|
||||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
STUCK_THRESHOLD_MINUTES = 5
|
STUCK_THRESHOLD_MINUTES = 5
|
||||||
|
# Archive ImportTasks run the per-member pipeline inline for every
|
||||||
|
# member (import_archive_file: soft=30min/hard=35min). The ImportTask
|
||||||
|
# 'processing' recovery sweep must give them a longer threshold or it
|
||||||
|
# re-queues a legitimately-running archive mid-import (double-process).
|
||||||
|
# 40 min = 5-min buffer past the archive task's hard kill.
|
||||||
|
# Operator-flagged 2026-05-28 (target 1645019, a big archive).
|
||||||
|
ARCHIVE_STUCK_THRESHOLD_MINUTES = 40
|
||||||
|
|
||||||
|
# Poison-pill cap. After being recovered (re-queued from a stuck
|
||||||
|
# 'processing' state) MAX_RECOVERY_ATTEMPTS-1 times, the next sweep
|
||||||
|
# marks the row 'failed' instead of looping. 3 = two recoveries then
|
||||||
|
# give up. A row reaches this only if it leaves NO terminal flip each
|
||||||
|
# run — i.e. it hard-crashes the worker (OOM/segfault/SIGKILL), the
|
||||||
|
# signature of a corrupt or oversized input. Caught exceptions already
|
||||||
|
# flip to terminal 'failed' and never enter this loop.
|
||||||
|
MAX_RECOVERY_ATTEMPTS = 3
|
||||||
ORPHAN_PENDING_THRESHOLD_MINUTES = 30
|
ORPHAN_PENDING_THRESHOLD_MINUTES = 30
|
||||||
|
|
||||||
|
# DownloadEvent (pending|running) recovery threshold. download_source has
|
||||||
|
# time_limit=1200s (20 min); 30 min is 10 min past that, so a legitimately-
|
||||||
|
# running task is never killed by the sweep. Operator-confirmed 2026-05-29
|
||||||
|
# after 43 sources stranded at "last check never" by the in-flight guard.
|
||||||
|
DOWNLOAD_STALL_THRESHOLD_MINUTES = 30
|
||||||
|
|
||||||
OLD_TASK_DAYS = 7
|
OLD_TASK_DAYS = 7
|
||||||
PHASH_PAGE = 500
|
PHASH_PAGE = 500
|
||||||
VERIFY_PAGE = 200
|
VERIFY_PAGE = 200
|
||||||
@@ -24,17 +55,40 @@ FFPROBE_TIMEOUT_SECONDS = 10
|
|||||||
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
||||||
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
||||||
|
|
||||||
|
# Overrides for recover_stalled_task_runs (the TaskRun 'running' sweep).
|
||||||
|
# Tasks/queues that legitimately run longer than the default 5-min
|
||||||
|
# threshold need their own larger value, else the sweep marks in-flight
|
||||||
|
# work 'error' before it finishes. Each value MUST be ≥ the relevant
|
||||||
|
# task.time_limit + a small buffer. task_name overrides take precedence
|
||||||
|
# over queue overrides.
|
||||||
|
#
|
||||||
|
# ml queue: tag_and_embed video branch (≈20 GPU ops); time_limit=1200.
|
||||||
|
# import_archive_file: shares the 'import' queue with the fast
|
||||||
|
# single-file import_media_file, so it needs a task-name override
|
||||||
|
# (the import queue itself stays at the 5-min default for single
|
||||||
|
# files); time_limit=2100.
|
||||||
|
QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||||
|
"ml": 25,
|
||||||
|
}
|
||||||
|
TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||||
|
"backend.app.tasks.import_file.import_archive_file": 40,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
|
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
|
||||||
def recover_interrupted_tasks() -> int:
|
def recover_interrupted_tasks() -> int:
|
||||||
"""Recover stuck ImportTask rows. Two distinct stuck states:
|
"""Recover stuck ImportTask rows. Two distinct stuck states:
|
||||||
|
|
||||||
1. 'processing' > 5 min — worker crash mid-import. Re-queue via
|
1. 'processing' too long — worker crash mid-import. Re-queue via
|
||||||
.delay() and let the import retry. Was 30 min historically;
|
enqueue_import (routing media vs archive) and let the import
|
||||||
tightened 2026-05-24 after operator hit a 2224-row zombie pile.
|
retry. Threshold is task-type-aware: media files are sub-second
|
||||||
import_media_file is sub-second for the vast majority of files and
|
and capped at the 5-min soft limit, so STUCK_THRESHOLD_MINUTES
|
||||||
capped at the per-task soft_time_limit (5 min), so anything still
|
(5) means a confirmed crash; archives run the per-member
|
||||||
'processing' after that window is a confirmed crash.
|
pipeline inline (import_archive_file, 35-min hard limit) so they
|
||||||
|
get ARCHIVE_STUCK_THRESHOLD_MINUTES (40) to avoid re-queueing a
|
||||||
|
still-running archive. (Media was tightened from 30 min to 5
|
||||||
|
2026-05-24 after a 2224-row zombie pile; archive split out
|
||||||
|
2026-05-28.)
|
||||||
|
|
||||||
2. 'pending' or 'queued' > 30 min — enqueue-phase crash. scan_directory
|
2. 'pending' or 'queued' > 30 min — enqueue-phase crash. scan_directory
|
||||||
creates rows with status='pending' (commit), then in a second pass
|
creates rows with status='pending' (commit), then in a second pass
|
||||||
@@ -51,7 +105,8 @@ def recover_interrupted_tasks() -> int:
|
|||||||
"""
|
"""
|
||||||
SessionLocal = _sync_session_factory()
|
SessionLocal = _sync_session_factory()
|
||||||
now = datetime.now(UTC)
|
now = datetime.now(UTC)
|
||||||
processing_cutoff = now - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
media_cutoff = now - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
||||||
|
archive_cutoff = now - timedelta(minutes=ARCHIVE_STUCK_THRESHOLD_MINUTES)
|
||||||
orphan_cutoff = now - timedelta(minutes=ORPHAN_PENDING_THRESHOLD_MINUTES)
|
orphan_cutoff = now - timedelta(minutes=ORPHAN_PENDING_THRESHOLD_MINUTES)
|
||||||
with SessionLocal() as session:
|
with SessionLocal() as session:
|
||||||
# Both sweeps used to be SELECT ids → UPDATE WHERE id IN (...) which
|
# Both sweeps used to be SELECT ids → UPDATE WHERE id IN (...) which
|
||||||
@@ -59,20 +114,67 @@ def recover_interrupted_tasks() -> int:
|
|||||||
# tens of thousands of rows (operator hit it 2026-05-26 after the
|
# tens of thousands of rows (operator hit it 2026-05-26 after the
|
||||||
# /import deep scan piled up orphans). Folding the SELECT into the
|
# /import deep scan piled up orphans). Folding the SELECT into the
|
||||||
# UPDATE eliminates the IN-list entirely. RETURNING gives us back
|
# UPDATE eliminates the IN-list entirely. RETURNING gives us back
|
||||||
# exactly the ids that flipped so the stuck sweep can still
|
# exactly the (id, task_type) pairs that flipped so the requeue
|
||||||
# .delay() each one.
|
# can route media vs archive correctly.
|
||||||
stuck_result = session.execute(
|
#
|
||||||
|
# Media + archive get separate cutoffs: a single media file is
|
||||||
|
# sub-second so 5 min means crash; an archive runs the per-member
|
||||||
|
# pipeline inline and can legitimately take up to its 35-min hard
|
||||||
|
# limit, so it gets ARCHIVE_STUCK_THRESHOLD_MINUTES (40) to avoid
|
||||||
|
# re-queueing a still-running archive.
|
||||||
|
stuck_predicate = and_(
|
||||||
|
ImportTask.status == "processing",
|
||||||
|
or_(
|
||||||
|
and_(ImportTask.task_type != "archive",
|
||||||
|
ImportTask.started_at < media_cutoff),
|
||||||
|
and_(ImportTask.task_type == "archive",
|
||||||
|
ImportTask.started_at < archive_cutoff),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# POISON-PILL CIRCUIT BREAKER (Layer 1, 2026-05-28). A row that
|
||||||
|
# leaves no terminal flip (hard worker crash: OOM/segfault/SIGKILL
|
||||||
|
# on a corrupt or oversized input) gets re-queued by this sweep —
|
||||||
|
# and would loop forever, re-crashing the worker each pass,
|
||||||
|
# without a cap. Once a row has already been recovered
|
||||||
|
# MAX_RECOVERY_ATTEMPTS-1 times, stop re-queueing it and mark it
|
||||||
|
# 'failed' with a diagnostic so the operator can find + replace
|
||||||
|
# the offending file. This UPDATE runs FIRST so the rows it
|
||||||
|
# claims drop out of 'processing' before the re-queue pass.
|
||||||
|
poison_result = session.execute(
|
||||||
update(ImportTask)
|
update(ImportTask)
|
||||||
.where(ImportTask.status == "processing")
|
.where(stuck_predicate)
|
||||||
.where(ImportTask.started_at < processing_cutoff)
|
.where(ImportTask.recovery_count >= MAX_RECOVERY_ATTEMPTS - 1)
|
||||||
.values(
|
.values(
|
||||||
status="queued",
|
status="failed",
|
||||||
started_at=None,
|
finished_at=now,
|
||||||
error="recovered from stuck state",
|
error=(
|
||||||
|
f"crashed or stalled the worker {MAX_RECOVERY_ATTEMPTS} "
|
||||||
|
f"times without completing — likely a corrupt or "
|
||||||
|
f"oversized input. Not re-queued. Inspect/replace the "
|
||||||
|
f"file, then retry via /api/import/retry-failed."
|
||||||
|
),
|
||||||
)
|
)
|
||||||
.returning(ImportTask.id)
|
.returning(ImportTask.id)
|
||||||
)
|
)
|
||||||
stuck_ids = [row[0] for row in stuck_result.all()]
|
poison_ids = [r[0] for r in poison_result.all()]
|
||||||
|
|
||||||
|
# Re-queue the remaining stuck rows (under the cap) and bump
|
||||||
|
# their recovery_count. RETURNING (id, task_type) so the requeue
|
||||||
|
# routes media vs archive correctly.
|
||||||
|
stuck_result = session.execute(
|
||||||
|
update(ImportTask)
|
||||||
|
.where(stuck_predicate)
|
||||||
|
.where(ImportTask.recovery_count < MAX_RECOVERY_ATTEMPTS - 1)
|
||||||
|
.values(
|
||||||
|
status="queued",
|
||||||
|
started_at=None,
|
||||||
|
recovery_count=ImportTask.recovery_count + 1,
|
||||||
|
error="recovered from stuck state",
|
||||||
|
)
|
||||||
|
.returning(ImportTask.id, ImportTask.task_type)
|
||||||
|
)
|
||||||
|
stuck = stuck_result.all()
|
||||||
|
|
||||||
orphan_result = session.execute(
|
orphan_result = session.execute(
|
||||||
update(ImportTask)
|
update(ImportTask)
|
||||||
@@ -91,12 +193,34 @@ def recover_interrupted_tasks() -> int:
|
|||||||
|
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
if stuck_ids:
|
if stuck:
|
||||||
from .import_file import import_media_file
|
from .import_file import enqueue_import
|
||||||
for tid in stuck_ids:
|
for tid, task_type in stuck:
|
||||||
import_media_file.delay(tid)
|
enqueue_import(tid, task_type)
|
||||||
|
|
||||||
return len(stuck_ids) + orphan_count
|
# Layer-2 auto re-download (env-gated, default OFF). For each
|
||||||
|
# poison-pill row that resolves to a pollable Source, delete the
|
||||||
|
# bad file and trigger ONE source re-check to fetch a fresh
|
||||||
|
# copy. Bounded by ImportTask.refetched so source-side
|
||||||
|
# corruption can't loop. The 'failed' row stays as history; the
|
||||||
|
# re-downloaded file re-imports as a fresh task on the next scan.
|
||||||
|
if poison_ids and os.environ.get("FC_AUTO_REFETCH_CORRUPT", "0") == "1":
|
||||||
|
from ..models import ImportSettings
|
||||||
|
from ..services.refetch_service import attempt_refetch
|
||||||
|
import_root = Path(session.execute(
|
||||||
|
select(ImportSettings.import_scan_path)
|
||||||
|
.where(ImportSettings.id == 1)
|
||||||
|
).scalar_one())
|
||||||
|
for pid in poison_ids:
|
||||||
|
ptask = session.get(ImportTask, pid)
|
||||||
|
if ptask is None:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
attempt_refetch(session, ptask, import_root)
|
||||||
|
except Exception as exc: # noqa: BLE001 — best-effort
|
||||||
|
log.warning("auto-refetch failed for task %s: %s", pid, exc)
|
||||||
|
|
||||||
|
return len(stuck) + len(poison_ids) + orphan_count
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks")
|
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks")
|
||||||
@@ -121,18 +245,29 @@ def cleanup_old_tasks() -> int:
|
|||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs")
|
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs")
|
||||||
def recover_stalled_task_runs() -> int:
|
def recover_stalled_task_runs() -> int:
|
||||||
"""Flip task_run rows stuck in 'running' for >STUCK_THRESHOLD_MINUTES
|
"""Flip task_run rows stuck in 'running' past their queue-specific
|
||||||
to 'error'. FC-3i.
|
threshold to 'error'. FC-3i.
|
||||||
|
|
||||||
A row gets stuck when the worker dies without emitting
|
A row gets stuck when the worker dies without emitting
|
||||||
task_postrun / task_failure (e.g. OOM, container restart between
|
task_postrun / task_failure (e.g. OOM, container restart between
|
||||||
signals, signal handler raised+logged). Shares the 5-min threshold
|
signals, signal handler raised+logged). The default 5-min threshold
|
||||||
with recover_interrupted_tasks for consistency.
|
fits short-lived queues (import/thumbnail/download); queues that
|
||||||
|
legitimately run longer tasks (ml-video, deep scans) get their
|
||||||
|
own larger threshold via QUEUE_STUCK_THRESHOLD_MINUTES so the
|
||||||
|
sweep doesn't preempt them.
|
||||||
|
|
||||||
|
Runs once per distinct threshold value: each pass updates rows
|
||||||
|
whose queue maps to that threshold.
|
||||||
"""
|
"""
|
||||||
SessionLocal = _sync_session_factory()
|
SessionLocal = _sync_session_factory()
|
||||||
cutoff = datetime.now(UTC) - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
now = datetime.now(UTC)
|
||||||
with SessionLocal() as session:
|
override_tasks = set(TASK_STUCK_THRESHOLD_MINUTES.keys())
|
||||||
result = session.execute(
|
override_queues = set(QUEUE_STUCK_THRESHOLD_MINUTES.keys())
|
||||||
|
total = 0
|
||||||
|
|
||||||
|
def _flag(minutes, *extra_where):
|
||||||
|
cutoff = now - timedelta(minutes=minutes)
|
||||||
|
stmt = (
|
||||||
update(TaskRun)
|
update(TaskRun)
|
||||||
.where(TaskRun.status == "running")
|
.where(TaskRun.status == "running")
|
||||||
.where(TaskRun.started_at < cutoff)
|
.where(TaskRun.started_at < cutoff)
|
||||||
@@ -140,14 +275,42 @@ def recover_stalled_task_runs() -> int:
|
|||||||
status="error",
|
status="error",
|
||||||
error_type="RecoverySweep",
|
error_type="RecoverySweep",
|
||||||
error_message=(
|
error_message=(
|
||||||
f"no completion signal received within "
|
f"no completion signal received within {minutes} min"
|
||||||
f"{STUCK_THRESHOLD_MINUTES} min"
|
|
||||||
),
|
),
|
||||||
finished_at=datetime.now(UTC),
|
finished_at=now,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
for w in extra_where:
|
||||||
|
stmt = stmt.where(w)
|
||||||
|
return session.execute(stmt).rowcount or 0
|
||||||
|
|
||||||
|
with SessionLocal() as session:
|
||||||
|
# Precedence: task_name override → queue override → default.
|
||||||
|
# Each pass excludes rows claimed by a higher-precedence pass so
|
||||||
|
# every row is touched at most once.
|
||||||
|
|
||||||
|
# 1. Per-task-name overrides (e.g. import_archive_file, which
|
||||||
|
# shares the 'import' queue with fast single-file imports).
|
||||||
|
for task_name, minutes in TASK_STUCK_THRESHOLD_MINUTES.items():
|
||||||
|
total += _flag(minutes, TaskRun.task_name == task_name)
|
||||||
|
|
||||||
|
# 2. Per-queue overrides, excluding the override task-names.
|
||||||
|
for queue, minutes in QUEUE_STUCK_THRESHOLD_MINUTES.items():
|
||||||
|
wheres = [TaskRun.queue == queue]
|
||||||
|
if override_tasks:
|
||||||
|
wheres.append(TaskRun.task_name.notin_(override_tasks))
|
||||||
|
total += _flag(minutes, *wheres)
|
||||||
|
|
||||||
|
# 3. Default — everything not claimed above.
|
||||||
|
default_wheres = []
|
||||||
|
if override_queues:
|
||||||
|
default_wheres.append(TaskRun.queue.notin_(override_queues))
|
||||||
|
if override_tasks:
|
||||||
|
default_wheres.append(TaskRun.task_name.notin_(override_tasks))
|
||||||
|
total += _flag(STUCK_THRESHOLD_MINUTES, *default_wheres)
|
||||||
|
|
||||||
session.commit()
|
session.commit()
|
||||||
return result.rowcount or 0
|
return total
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.prune_task_runs")
|
@celery.task(name="backend.app.tasks.maintenance.prune_task_runs")
|
||||||
@@ -299,6 +462,65 @@ def verify_integrity() -> int:
|
|||||||
return total
|
return total
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_download_events")
|
||||||
|
def recover_stalled_download_events() -> int:
|
||||||
|
"""Recover DownloadEvent rows stuck pending/running past the worker hard kill.
|
||||||
|
|
||||||
|
The scan tick (scheduler_service.select_due_sources →
|
||||||
|
tasks.scan._tick_due_sources_async) inserts DownloadEvent(status='pending')
|
||||||
|
and fires download_source.delay(). If that task dies before finalizing the
|
||||||
|
event — worker OOM/SIGKILL, lost task, or a gallery-dl that didn't unwind
|
||||||
|
on the 1200s hard time_limit — the event stays in-flight forever. The next
|
||||||
|
tick then skips that source because of the in-flight guard (scan.py:168)
|
||||||
|
and Source.last_checked_at never updates; the operator sees "last check
|
||||||
|
never" in the Subscriptions health column, permanently.
|
||||||
|
|
||||||
|
This sweep flips matching events to 'error', stamps each affected Source's
|
||||||
|
last_checked_at + last_error and bumps consecutive_failures (once per
|
||||||
|
source, not per event — backoff is exponential on that count so an N-event
|
||||||
|
bump would inflate the next interval by 2^N for no reason). The source
|
||||||
|
becomes re-queueable on the next tick and the health dot goes amber.
|
||||||
|
|
||||||
|
Operator-confirmed 2026-05-29 (43-row strand pile in production).
|
||||||
|
"""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
cutoff = now - timedelta(minutes=DOWNLOAD_STALL_THRESHOLD_MINUTES)
|
||||||
|
msg = "stranded by recovery sweep (no terminal status after time_limit)"
|
||||||
|
with SessionLocal() as session:
|
||||||
|
# UPDATE...RETURNING the source_ids in one round trip — keeps us off
|
||||||
|
# the psycopg 65535-param ceiling that SELECT-then-UPDATE-WHERE-IN
|
||||||
|
# would hit on a large strand pile.
|
||||||
|
result = session.execute(
|
||||||
|
update(DownloadEvent)
|
||||||
|
.where(DownloadEvent.status.in_(["pending", "running"]))
|
||||||
|
.where(DownloadEvent.started_at < cutoff)
|
||||||
|
.values(status="error", finished_at=now, error=msg)
|
||||||
|
.returning(DownloadEvent.source_id)
|
||||||
|
)
|
||||||
|
returned = result.all()
|
||||||
|
if not returned:
|
||||||
|
session.commit()
|
||||||
|
return 0
|
||||||
|
events_recovered = len(returned)
|
||||||
|
source_ids = list({row.source_id for row in returned})
|
||||||
|
session.execute(
|
||||||
|
update(Source)
|
||||||
|
.where(Source.id.in_(source_ids))
|
||||||
|
.values(
|
||||||
|
consecutive_failures=Source.consecutive_failures + 1,
|
||||||
|
last_error=msg,
|
||||||
|
last_checked_at=now,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
log.info(
|
||||||
|
"recover_stalled_download_events: recovered %d events across %d sources",
|
||||||
|
events_recovered, len(source_ids),
|
||||||
|
)
|
||||||
|
return events_recovered
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_download_events")
|
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_download_events")
|
||||||
def cleanup_old_download_events() -> int:
|
def cleanup_old_download_events() -> int:
|
||||||
"""FC-3d: delete terminal DownloadEvent rows older than the configured
|
"""FC-3d: delete terminal DownloadEvent rows older than the configured
|
||||||
@@ -311,9 +533,7 @@ def cleanup_old_download_events() -> int:
|
|||||||
"""
|
"""
|
||||||
SessionLocal = _sync_session_factory()
|
SessionLocal = _sync_session_factory()
|
||||||
with SessionLocal() as session:
|
with SessionLocal() as session:
|
||||||
settings = session.execute(
|
settings = ImportSettings.load_sync(session)
|
||||||
select(ImportSettings).where(ImportSettings.id == 1)
|
|
||||||
).scalar_one()
|
|
||||||
retention_days = settings.download_event_retention_days
|
retention_days = settings.download_event_retention_days
|
||||||
cutoff = datetime.now(UTC) - timedelta(days=retention_days)
|
cutoff = datetime.now(UTC) - timedelta(days=retention_days)
|
||||||
result = session.execute(
|
result = session.execute(
|
||||||
|
|||||||
@@ -1,175 +0,0 @@
|
|||||||
"""FC-5 run_migration Celery task.
|
|
||||||
|
|
||||||
Dispatches to the right migrator based on `kind`. Updates MigrationRun
|
|
||||||
row's status/counts/finished_at as it runs. Failures set status='error'
|
|
||||||
with the error message preserved.
|
|
||||||
|
|
||||||
kinds: gs_ingest, ir_ingest, tag_apply, ml_queue, verify, cleanup
|
|
||||||
(backup + rollback retired 2026-05-24 → see /api/system/backup/*)
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import logging
|
|
||||||
from datetime import UTC, datetime
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
||||||
|
|
||||||
from ..celery_app import celery
|
|
||||||
from ..config import get_config
|
|
||||||
from ..models import MigrationRun
|
|
||||||
from ..services.credential_crypto import CredentialCrypto
|
|
||||||
from ..services.migrators import cleanup as cleanup_mod
|
|
||||||
from ..services.migrators import gs_ingest, ir_ingest, ml_queue, tag_apply, verify
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
IMAGES_ROOT = Path("/images")
|
|
||||||
_KEY_PATH = IMAGES_ROOT / "secrets" / "credential_key.b64"
|
|
||||||
|
|
||||||
|
|
||||||
def _async_session_factory():
|
|
||||||
cfg = get_config()
|
|
||||||
engine = create_async_engine(cfg.database_url, future=True, pool_pre_ping=True)
|
|
||||||
return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False), engine
|
|
||||||
|
|
||||||
|
|
||||||
async def _update_run(
|
|
||||||
db: AsyncSession, run_id: int, *,
|
|
||||||
status: str | None = None, counts: dict | None = None,
|
|
||||||
error: str | None = None, finished_at: datetime | None = None,
|
|
||||||
metadata_patch: dict | None = None,
|
|
||||||
) -> None:
|
|
||||||
run = (await db.execute(
|
|
||||||
select(MigrationRun).where(MigrationRun.id == run_id)
|
|
||||||
)).scalar_one()
|
|
||||||
if status is not None:
|
|
||||||
run.status = status
|
|
||||||
if counts is not None:
|
|
||||||
run.counts = counts
|
|
||||||
if error is not None:
|
|
||||||
run.error = error
|
|
||||||
if finished_at is not None:
|
|
||||||
run.finished_at = finished_at
|
|
||||||
if metadata_patch:
|
|
||||||
run.metadata_ = {**(run.metadata_ or {}), **metadata_patch}
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
|
|
||||||
async def _run_async(run_id: int, kind: str, params: dict) -> dict:
|
|
||||||
factory, engine = _async_session_factory()
|
|
||||||
try:
|
|
||||||
async with factory() as db:
|
|
||||||
await _update_run(db, run_id, status="running")
|
|
||||||
try:
|
|
||||||
if kind in ("backup", "rollback"):
|
|
||||||
raise ValueError(
|
|
||||||
f"kind {kind!r} retired in FC-3h; "
|
|
||||||
"use /api/system/backup/* instead"
|
|
||||||
)
|
|
||||||
|
|
||||||
elif kind == "gs_ingest":
|
|
||||||
fc_crypto = CredentialCrypto(_KEY_PATH)
|
|
||||||
counts = await gs_ingest.migrate_async(
|
|
||||||
db, data=params["data"],
|
|
||||||
fc_crypto=fc_crypto,
|
|
||||||
dry_run=params.get("dry_run", False),
|
|
||||||
)
|
|
||||||
await _update_run(
|
|
||||||
db, run_id, status="ok", counts=counts,
|
|
||||||
finished_at=datetime.now(UTC),
|
|
||||||
)
|
|
||||||
return counts
|
|
||||||
|
|
||||||
elif kind == "ir_ingest":
|
|
||||||
counts = await ir_ingest.migrate_async(
|
|
||||||
db, data=params["data"],
|
|
||||||
images_root=IMAGES_ROOT,
|
|
||||||
dry_run=params.get("dry_run", False),
|
|
||||||
)
|
|
||||||
await _update_run(
|
|
||||||
db, run_id, status="ok", counts=counts,
|
|
||||||
finished_at=datetime.now(UTC),
|
|
||||||
)
|
|
||||||
return counts
|
|
||||||
|
|
||||||
elif kind == "tag_apply":
|
|
||||||
result = await tag_apply.apply_async(
|
|
||||||
db, images_root=IMAGES_ROOT,
|
|
||||||
dry_run=params.get("dry_run", False),
|
|
||||||
)
|
|
||||||
await _update_run(
|
|
||||||
db, run_id, status="ok",
|
|
||||||
counts=result["counts"],
|
|
||||||
finished_at=datetime.now(UTC),
|
|
||||||
metadata_patch={"unmatched": result["unmatched"]},
|
|
||||||
)
|
|
||||||
return result
|
|
||||||
|
|
||||||
elif kind == "ml_queue":
|
|
||||||
count = await ml_queue.queue_all_unprocessed_async(db)
|
|
||||||
await _update_run(
|
|
||||||
db, run_id, status="ok",
|
|
||||||
counts={"rows_processed": count, "rows_inserted": 0,
|
|
||||||
"rows_skipped": 0, "files_copied": 0,
|
|
||||||
"bytes_copied": 0, "conflicts": 0},
|
|
||||||
finished_at=datetime.now(UTC),
|
|
||||||
)
|
|
||||||
return {"queued": count}
|
|
||||||
|
|
||||||
elif kind == "verify":
|
|
||||||
checks = await verify.verify_async(db, expected=params.get("expected"))
|
|
||||||
sample = await verify.verify_sha256_sample(
|
|
||||||
db, sample_size=params.get("sample_size", 20),
|
|
||||||
)
|
|
||||||
await _update_run(
|
|
||||||
db, run_id, status="ok",
|
|
||||||
counts={"rows_processed": sample["sample_size"],
|
|
||||||
"rows_inserted": 0, "rows_skipped": 0,
|
|
||||||
"files_copied": 0, "bytes_copied": 0,
|
|
||||||
"conflicts": sample["mismatched"] + sample["missing"]},
|
|
||||||
finished_at=datetime.now(UTC),
|
|
||||||
metadata_patch={"checks": checks, "sample": sample},
|
|
||||||
)
|
|
||||||
return {"checks": checks, "sample": sample}
|
|
||||||
|
|
||||||
elif kind == "cleanup":
|
|
||||||
slug = params.get("slug")
|
|
||||||
if not slug:
|
|
||||||
raise ValueError("cleanup requires params.slug")
|
|
||||||
result = await cleanup_mod.cleanup_artist_async(
|
|
||||||
db, slug=slug, images_root=IMAGES_ROOT,
|
|
||||||
dry_run=params.get("dry_run", False),
|
|
||||||
source_path_prefix=params.get("source_path_prefix"),
|
|
||||||
)
|
|
||||||
await _update_run(
|
|
||||||
db, run_id, status="ok",
|
|
||||||
counts=result["counts"],
|
|
||||||
finished_at=datetime.now(UTC),
|
|
||||||
metadata_patch={
|
|
||||||
"artist": result["artist"],
|
|
||||||
"summary": result["summary"],
|
|
||||||
},
|
|
||||||
)
|
|
||||||
return result
|
|
||||||
|
|
||||||
else:
|
|
||||||
raise ValueError(f"unknown kind: {kind}")
|
|
||||||
|
|
||||||
except Exception as exc:
|
|
||||||
log.exception("migration kind=%s failed", kind)
|
|
||||||
await _update_run(
|
|
||||||
db, run_id, status="error", error=str(exc),
|
|
||||||
finished_at=datetime.now(UTC),
|
|
||||||
)
|
|
||||||
raise
|
|
||||||
finally:
|
|
||||||
await engine.dispose()
|
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.migration.run_migration", bind=True, acks_late=True)
|
|
||||||
def run_migration(self, run_id: int, kind: str, params: dict) -> dict:
|
|
||||||
"""FC-5: dispatch a migration kind. Updates MigrationRun row as it goes."""
|
|
||||||
return asyncio.run(_run_async(run_id, kind, params))
|
|
||||||
+21
-2
@@ -31,8 +31,15 @@ def _is_video(path: Path) -> bool:
|
|||||||
retry_backoff_max=60,
|
retry_backoff_max=60,
|
||||||
retry_jitter=True,
|
retry_jitter=True,
|
||||||
max_retries=3,
|
max_retries=3,
|
||||||
soft_time_limit=300,
|
# Sized for the video branch: sample 10 frames, run tagger +
|
||||||
time_limit=420,
|
# embedder on each (≈20 GPU ops vs 2 for an image). A loaded
|
||||||
|
# ml-worker can take 5-10 min on a long video; bumped from
|
||||||
|
# 5min/7min on 2026-05-28 after operator-flagged image 6288 (a
|
||||||
|
# .mp4) hit the recovery sweep at 5 min while still legitimately
|
||||||
|
# processing. Image runs return in seconds; the bump doesn't
|
||||||
|
# affect their UX.
|
||||||
|
soft_time_limit=900, # 15 min
|
||||||
|
time_limit=1200, # 20 min hard
|
||||||
)
|
)
|
||||||
def tag_and_embed(self, image_id: int) -> dict:
|
def tag_and_embed(self, image_id: int) -> dict:
|
||||||
"""Run Camie + SigLIP on one image; store predictions + embedding;
|
"""Run Camie + SigLIP on one image; store predictions + embedding;
|
||||||
@@ -64,6 +71,18 @@ def tag_and_embed(self, image_id: int) -> dict:
|
|||||||
embedder = get_embedder()
|
embedder = get_embedder()
|
||||||
|
|
||||||
if _is_video(src):
|
if _is_video(src):
|
||||||
|
# Layer-3 isolation: ffprobe (a separate process) validates
|
||||||
|
# the container before we burn ~20 GPU ops sampling frames
|
||||||
|
# from it. A corrupt video that would crash the frame
|
||||||
|
# decoder is rejected cleanly here instead of taking down
|
||||||
|
# the ml-worker. Operator-flagged 2026-05-28.
|
||||||
|
from ..utils import safe_probe
|
||||||
|
vprobe = safe_probe.probe_video(src)
|
||||||
|
if not vprobe.ok:
|
||||||
|
return {
|
||||||
|
"status": "bad_video", "image_id": image_id,
|
||||||
|
"reason": vprobe.reason,
|
||||||
|
}
|
||||||
frames = _sample_video_frames(
|
frames = _sample_video_frames(
|
||||||
src, int(os.environ.get("VIDEO_ML_FRAMES", "10"))
|
src, int(os.environ.get("VIDEO_ML_FRAMES", "10"))
|
||||||
)
|
)
|
||||||
|
|||||||
+15
-18
@@ -12,12 +12,12 @@ from datetime import UTC, datetime
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
||||||
|
|
||||||
from ..celery_app import celery
|
from ..celery_app import celery
|
||||||
from ..config import get_config
|
|
||||||
from ..models import DownloadEvent, ImportBatch, ImportSettings, ImportTask
|
from ..models import DownloadEvent, ImportBatch, ImportSettings, ImportTask
|
||||||
from ..services.scheduler_service import select_due_sources
|
from ..services.archive_extractor import is_archive
|
||||||
|
from ..services.scheduler_service import record_tick, select_due_sources
|
||||||
|
from ._async_session import async_session_factory
|
||||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||||
|
|
||||||
|
|
||||||
@@ -44,9 +44,7 @@ def scan_directory(self, triggered_by: str = "manual",
|
|||||||
batch id."""
|
batch id."""
|
||||||
SessionLocal = _sync_session_factory()
|
SessionLocal = _sync_session_factory()
|
||||||
with SessionLocal() as session:
|
with SessionLocal() as session:
|
||||||
settings = session.execute(
|
settings = ImportSettings.load_sync(session)
|
||||||
select(ImportSettings).where(ImportSettings.id == 1)
|
|
||||||
).scalar_one()
|
|
||||||
import_root = Path(settings.import_scan_path)
|
import_root = Path(settings.import_scan_path)
|
||||||
|
|
||||||
batch = ImportBatch(
|
batch = ImportBatch(
|
||||||
@@ -96,7 +94,9 @@ def scan_directory(self, triggered_by: str = "manual",
|
|||||||
task = ImportTask(
|
task = ImportTask(
|
||||||
batch_id=batch_id,
|
batch_id=batch_id,
|
||||||
source_path=entry_str,
|
source_path=entry_str,
|
||||||
task_type="media",
|
# Archives route to import_archive_file (larger time
|
||||||
|
# budget) — they run the per-member pipeline inline.
|
||||||
|
task_type="archive" if is_archive(entry) else "media",
|
||||||
status="pending",
|
status="pending",
|
||||||
size_bytes=size,
|
size_bytes=size,
|
||||||
)
|
)
|
||||||
@@ -115,15 +115,16 @@ def scan_directory(self, triggered_by: str = "manual",
|
|||||||
batch.finished_at = datetime.now(UTC)
|
batch.finished_at = datetime.now(UTC)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
# Now enqueue import_media_file for each pending task.
|
# Now enqueue each pending task on the right Celery task
|
||||||
|
# (media vs archive) via the shared router.
|
||||||
|
from .import_file import enqueue_import
|
||||||
|
|
||||||
for task in session.execute(
|
for task in session.execute(
|
||||||
select(ImportTask).where(ImportTask.batch_id == batch_id)
|
select(ImportTask).where(ImportTask.batch_id == batch_id)
|
||||||
).scalars():
|
).scalars():
|
||||||
task.status = "queued"
|
task.status = "queued"
|
||||||
session.add(task)
|
session.add(task)
|
||||||
from .import_file import import_media_file
|
enqueue_import(task.id, task.task_type)
|
||||||
|
|
||||||
import_media_file.delay(task.id)
|
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
if mode == "deep":
|
if mode == "deep":
|
||||||
@@ -137,16 +138,12 @@ def scan_directory(self, triggered_by: str = "manual",
|
|||||||
# --- FC-3d: periodic source-check tick ------------------------------------
|
# --- FC-3d: periodic source-check tick ------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def _async_session_factory():
|
|
||||||
cfg = get_config()
|
|
||||||
engine = create_async_engine(cfg.database_url, future=True, pool_pre_ping=True)
|
|
||||||
return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False), engine
|
|
||||||
|
|
||||||
|
|
||||||
async def _tick_due_sources_async() -> dict:
|
async def _tick_due_sources_async() -> dict:
|
||||||
factory, engine = _async_session_factory()
|
factory, engine = async_session_factory()
|
||||||
try:
|
try:
|
||||||
async with factory() as session:
|
async with factory() as session:
|
||||||
|
# Prove the scheduler is alive even on empty ticks (UI reads this).
|
||||||
|
await record_tick(session)
|
||||||
due = await select_due_sources(session)
|
due = await select_due_sources(session)
|
||||||
if not due:
|
if not due:
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
"""Subprocess entrypoint for safe_probe.probe_archive — see safe_probe.py.
|
||||||
|
|
||||||
|
probe_archive spawns this via subprocess (not multiprocessing.Process)
|
||||||
|
because Celery's prefork worker pool runs tasks in DAEMON processes and
|
||||||
|
Python's multiprocessing forbids daemon processes from spawning children
|
||||||
|
("AssertionError: daemonic processes are not allowed to have children",
|
||||||
|
operator-flagged 2026-05-30 — every archive import failed at task
|
||||||
|
startup). subprocess has no such restriction; we still get crash-
|
||||||
|
isolation because a probe segfault/OOM exits non-zero rather than
|
||||||
|
killing the worker.
|
||||||
|
|
||||||
|
Prints a single JSON line on stdout: {"status": "ok"|"error",
|
||||||
|
"detail": "..."?}. Exit code 0 for clean outcomes; non-zero exit
|
||||||
|
(signal / OOM-kill / unhandled exception) is the poison-pill signature
|
||||||
|
the parent maps to ProbeResult(crashed=True).
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from .safe_probe import _run_probe
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
if len(sys.argv) != 2:
|
||||||
|
print(json.dumps({"status": "error", "detail": "usage: <path>"}))
|
||||||
|
return 2
|
||||||
|
status, detail = _run_probe(sys.argv[1])
|
||||||
|
print(json.dumps({"status": status, "detail": detail}))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,191 @@
|
|||||||
|
"""Subprocess-isolated media probes (Layer 3 of import resilience).
|
||||||
|
|
||||||
|
A malformed video or archive can hard-crash the worker process — a
|
||||||
|
decoder OOM, a native-lib segfault, or a decompression bomb. A hard
|
||||||
|
crash leaves no terminal flip, so the recovery sweep re-queues the row
|
||||||
|
and it crashes again: a poison-pill loop (the Layer-1 cap is the
|
||||||
|
backstop, but isolating the crash is better — the file gets a clean
|
||||||
|
terminal failure and the worker never dies).
|
||||||
|
|
||||||
|
These probes run the risky read in a way that contains the blast:
|
||||||
|
|
||||||
|
- Video: `ffprobe` is a separate binary, so a crash decoding the
|
||||||
|
container kills only ffprobe (non-zero exit), never the worker. Also
|
||||||
|
returns width/height, which the importer didn't previously capture
|
||||||
|
for videos.
|
||||||
|
- Archive: an uncompressed-size guard (catches decompression bombs
|
||||||
|
before they OOM anything) plus an integrity test in a spawned child
|
||||||
|
(catches native-lib crashes on a malformed archive). A child segfault
|
||||||
|
/ OOM shows up as a non-zero exit code, not a dead worker.
|
||||||
|
|
||||||
|
Images are intentionally NOT probed here: Pillow raises (it doesn't
|
||||||
|
segfault) on the realistic corrupt-image cases, the importer already
|
||||||
|
catches that as an invalid_image skip, and a subprocess per image would
|
||||||
|
wreck deep-scan throughput on a large library. Add an image branch only
|
||||||
|
if a real image-induced worker crash is ever observed.
|
||||||
|
|
||||||
|
Operator-requested 2026-05-28 (Layer 3).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
VIDEO_PROBE_TIMEOUT_SECONDS = 60
|
||||||
|
ARCHIVE_PROBE_TIMEOUT_SECONDS = 120
|
||||||
|
# Refuse archives whose total UNCOMPRESSED size exceeds this — the
|
||||||
|
# classic decompression-bomb guard (a 4 GB cap comfortably clears real
|
||||||
|
# art-pack archives while stopping a few-KB zip that expands to TB).
|
||||||
|
MAX_ARCHIVE_UNCOMPRESSED_BYTES = 4 * 1024 * 1024 * 1024
|
||||||
|
|
||||||
|
# Repo root for the subprocess cwd so `python -m backend.app.utils.*`
|
||||||
|
# resolves regardless of where Celery / pytest started. backend/app/utils
|
||||||
|
# = parents[0]; backend/app = parents[1]; backend = parents[2]; repo root
|
||||||
|
# = parents[3].
|
||||||
|
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||||
|
_PROBE_RUNNER_MODULE = "backend.app.utils._archive_probe_runner"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ProbeResult:
|
||||||
|
ok: bool
|
||||||
|
# crashed=True means the probe HARD-FAILED (subprocess killed by a
|
||||||
|
# signal, OOM, or timeout) — the poison-pill signature. crashed=False
|
||||||
|
# with ok=False means a clean rejection (corrupt-but-handled,
|
||||||
|
# bomb-size-exceeded, integrity mismatch). Callers map crashed → a
|
||||||
|
# terminal 'failed', clean → a 'skipped'/'failed' of their choosing.
|
||||||
|
crashed: bool = False
|
||||||
|
reason: str | None = None
|
||||||
|
width: int | None = None
|
||||||
|
height: int | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def probe_video(path: Path, *, timeout: float = VIDEO_PROBE_TIMEOUT_SECONDS) -> ProbeResult:
|
||||||
|
"""Validate a video container + first video stream via ffprobe."""
|
||||||
|
try:
|
||||||
|
out = subprocess.run(
|
||||||
|
[
|
||||||
|
"ffprobe", "-v", "error",
|
||||||
|
"-select_streams", "v:0",
|
||||||
|
"-show_entries", "stream=width,height",
|
||||||
|
"-of", "json", str(path),
|
||||||
|
],
|
||||||
|
capture_output=True, text=True, timeout=timeout,
|
||||||
|
)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
return ProbeResult(ok=False, crashed=True, reason="ffprobe timed out")
|
||||||
|
except OSError as exc:
|
||||||
|
# ffprobe missing / not executable — environmental, not the
|
||||||
|
# file's fault. Treat as a clean non-crash failure so the import
|
||||||
|
# path can decide (it currently proceeds without dims).
|
||||||
|
return ProbeResult(ok=False, crashed=False, reason=f"ffprobe unavailable: {exc}")
|
||||||
|
if out.returncode != 0:
|
||||||
|
return ProbeResult(
|
||||||
|
ok=False, crashed=False,
|
||||||
|
reason=f"ffprobe rejected the file: {out.stderr.strip()[:200]}",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
streams = (json.loads(out.stdout) or {}).get("streams") or []
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
return ProbeResult(ok=False, crashed=False, reason=f"ffprobe output parse failed: {exc}")
|
||||||
|
if not streams:
|
||||||
|
return ProbeResult(ok=False, crashed=False, reason="no decodable video stream")
|
||||||
|
return ProbeResult(
|
||||||
|
ok=True, width=streams[0].get("width"), height=streams[0].get("height"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def probe_archive(path: Path, *, timeout: float = ARCHIVE_PROBE_TIMEOUT_SECONDS) -> ProbeResult:
|
||||||
|
"""Bomb-size guard + isolated integrity test for an archive.
|
||||||
|
|
||||||
|
Runs via subprocess (not multiprocessing.Process) because Celery's
|
||||||
|
prefork worker pool is daemon-mode and Python's multiprocessing
|
||||||
|
forbids daemon processes from spawning children ("AssertionError:
|
||||||
|
daemonic processes are not allowed to have children"). subprocess
|
||||||
|
has no such restriction and still gives the crash isolation: a probe
|
||||||
|
segfault/OOM exits non-zero rather than killing the worker.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
[sys.executable, "-m", _PROBE_RUNNER_MODULE, str(path)],
|
||||||
|
capture_output=True, text=True, timeout=timeout,
|
||||||
|
cwd=str(_REPO_ROOT),
|
||||||
|
)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
return ProbeResult(ok=False, crashed=True, reason="archive probe timed out")
|
||||||
|
if result.returncode != 0:
|
||||||
|
# Negative = killed by signal (segfault); positive = unhandled
|
||||||
|
# exception or OOM-kill. Either way: poison-pill signature.
|
||||||
|
return ProbeResult(
|
||||||
|
ok=False, crashed=True,
|
||||||
|
reason=f"archive probe crashed (exit {result.returncode})",
|
||||||
|
)
|
||||||
|
last_line = result.stdout.strip().splitlines()[-1:] or [""]
|
||||||
|
try:
|
||||||
|
outcome = json.loads(last_line[0])
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
return ProbeResult(
|
||||||
|
ok=False, crashed=True,
|
||||||
|
reason=f"archive probe produced no parseable result: {exc}",
|
||||||
|
)
|
||||||
|
if outcome.get("status") == "ok":
|
||||||
|
return ProbeResult(ok=True)
|
||||||
|
return ProbeResult(
|
||||||
|
ok=False, crashed=False,
|
||||||
|
reason=outcome.get("detail") or "archive probe rejected",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_probe(path_str: str) -> tuple[str, str | None]:
|
||||||
|
"""Pure-Python body of the archive probe — bomb-guard + integrity test.
|
||||||
|
|
||||||
|
Returns ('ok', None) or ('error', reason). Caught exceptions become
|
||||||
|
clean 'error' rejections; uncaught crashes in the subprocess become
|
||||||
|
non-zero exit codes (poison-pill signature) handled by probe_archive.
|
||||||
|
|
||||||
|
Exposed at the module level so the subprocess runner and tests both
|
||||||
|
call the same code path.
|
||||||
|
"""
|
||||||
|
path = Path(path_str)
|
||||||
|
ext = path.suffix.lower()
|
||||||
|
try:
|
||||||
|
total, test_bad = _inspect_archive(path, ext)
|
||||||
|
except Exception as exc: # noqa: BLE001 — clean rejection
|
||||||
|
return ("error", f"{type(exc).__name__}: {exc}")
|
||||||
|
if total is not None and total > MAX_ARCHIVE_UNCOMPRESSED_BYTES:
|
||||||
|
gib = total / (1024 ** 3)
|
||||||
|
return ("error", f"uncompressed size {gib:.1f} GiB exceeds the bomb-guard cap")
|
||||||
|
if test_bad is not None:
|
||||||
|
return ("error", f"integrity test failed at member {test_bad!r}")
|
||||||
|
return ("ok", None)
|
||||||
|
|
||||||
|
|
||||||
|
def _inspect_archive(path: Path, ext: str):
|
||||||
|
"""Return (total_uncompressed_bytes | None, first_bad_member | None)
|
||||||
|
for the archive. Format-specific; raises on a structurally-broken
|
||||||
|
container (caught by the child as a clean rejection)."""
|
||||||
|
if ext in (".zip", ".cbz"):
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
with zipfile.ZipFile(path) as zf:
|
||||||
|
total = sum(zi.file_size for zi in zf.infolist())
|
||||||
|
return total, zf.testzip()
|
||||||
|
if ext == ".rar":
|
||||||
|
import rarfile
|
||||||
|
|
||||||
|
with rarfile.RarFile(path) as rf:
|
||||||
|
total = sum(getattr(ri, "file_size", 0) for ri in rf.infolist())
|
||||||
|
rf.testrar()
|
||||||
|
return total, None
|
||||||
|
if ext == ".7z":
|
||||||
|
import py7zr
|
||||||
|
|
||||||
|
with py7zr.SevenZipFile(path, "r") as zf:
|
||||||
|
info = zf.archiveinfo()
|
||||||
|
total = getattr(info, "uncompressed", None)
|
||||||
|
ok = zf.test() # True / None when all members pass
|
||||||
|
return total, (None if ok in (True, None) else "7z test reported corruption")
|
||||||
|
# Unknown extension — nothing to test; treat as clean.
|
||||||
|
return None, None
|
||||||
@@ -259,6 +259,29 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
|||||||
return { error: e.message };
|
return { error: e.message };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case 'PROBE_SOURCE':
|
||||||
|
try {
|
||||||
|
return await api.probeSource(msg.url);
|
||||||
|
} catch (e) {
|
||||||
|
return { error: e.message };
|
||||||
|
}
|
||||||
|
|
||||||
|
case 'OPEN_ARTIST_PAGE': {
|
||||||
|
// apiUrl is configured with the /api suffix (see
|
||||||
|
// options/options.html placeholder); the SPA artist route is
|
||||||
|
// /artist/:slug, served from the same origin. Strip /api so the
|
||||||
|
// browser-level URL hits the Vue router, not the JSON API.
|
||||||
|
const base = (api.baseUrl || '').replace(/\/+$/, '').replace(/\/api$/, '');
|
||||||
|
const slug = encodeURIComponent(msg.slug || '');
|
||||||
|
if (!base || !slug) return { error: 'apiUrl or slug missing' };
|
||||||
|
try {
|
||||||
|
await browser.tabs.create({ url: `${base}/artist/${slug}` });
|
||||||
|
return { success: true };
|
||||||
|
} catch (e) {
|
||||||
|
return { error: e.message };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return { error: `Unknown message type: ${msg.type}` };
|
return { error: `Unknown message type: ${msg.type}` };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,11 +5,26 @@
|
|||||||
background: rgb(20, 23, 26); color: rgb(244, 186, 122);
|
background: rgb(20, 23, 26); color: rgb(244, 186, 122);
|
||||||
font: 500 14px/1.2 system-ui, sans-serif;
|
font: 500 14px/1.2 system-ui, sans-serif;
|
||||||
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.4); cursor: pointer;
|
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.4); cursor: pointer;
|
||||||
transition: transform 100ms ease;
|
transition: transform 100ms ease, background 150ms ease, color 150ms ease;
|
||||||
}
|
}
|
||||||
.fc-add-source-btn:hover { transform: translateY(-1px); }
|
.fc-add-source-btn:hover { transform: translateY(-1px); }
|
||||||
.fc-add-source-btn:disabled { opacity: 0.6; cursor: wait; }
|
.fc-add-source-btn:disabled { opacity: 0.6; cursor: wait; }
|
||||||
|
|
||||||
|
/* state colors map to the FC palette: parchment-on-slate base,
|
||||||
|
accent-orange for new, sage for already-subscribed, amber-warning for
|
||||||
|
artist-exists-but-source-missing. All readable on the dark base. */
|
||||||
|
.fc-add-source-btn--new {
|
||||||
|
background: rgb(20, 23, 26); color: rgb(244, 186, 122);
|
||||||
|
}
|
||||||
|
.fc-add-source-btn--artist-match {
|
||||||
|
background: rgb(28, 23, 16); color: rgb(255, 200, 120);
|
||||||
|
border: 1px solid rgb(180, 130, 60);
|
||||||
|
}
|
||||||
|
.fc-add-source-btn--source-match {
|
||||||
|
background: rgb(18, 28, 20); color: rgb(140, 220, 160);
|
||||||
|
border: 1px solid rgb(80, 160, 100);
|
||||||
|
}
|
||||||
|
|
||||||
.fc-toast {
|
.fc-toast {
|
||||||
all: revert;
|
all: revert;
|
||||||
position: fixed; bottom: 84px; right: 24px; z-index: 2147483647;
|
position: fixed; bottom: 84px; right: 24px; z-index: 2147483647;
|
||||||
|
|||||||
@@ -2,6 +2,10 @@
|
|||||||
if (window.__fc_addsource_injected) return;
|
if (window.__fc_addsource_injected) return;
|
||||||
window.__fc_addsource_injected = true;
|
window.__fc_addsource_injected = true;
|
||||||
|
|
||||||
|
// Cached probe result for the current URL so click-handlers know which
|
||||||
|
// action to dispatch without round-tripping again.
|
||||||
|
let currentProbe = null;
|
||||||
|
|
||||||
evaluate();
|
evaluate();
|
||||||
|
|
||||||
const reEval = () => evaluate();
|
const reEval = () => evaluate();
|
||||||
@@ -9,38 +13,116 @@
|
|||||||
const origPush = history.pushState;
|
const origPush = history.pushState;
|
||||||
history.pushState = function () { origPush.apply(this, arguments); reEval(); };
|
history.pushState = function () { origPush.apply(this, arguments); reEval(); };
|
||||||
|
|
||||||
function evaluate() {
|
async function evaluate() {
|
||||||
const platform = getPlatformFromUrl(window.location.href);
|
const url = window.location.href;
|
||||||
const onArtist = platform && isArtistPage(window.location.href, platform);
|
const platform = getPlatformFromUrl(url);
|
||||||
let btn = document.getElementById('fc-add-source-btn');
|
const onArtist = platform && isArtistPage(url, platform);
|
||||||
if (onArtist && !btn) injectButton();
|
const btn = document.getElementById('fc-add-source-btn');
|
||||||
else if (!onArtist && btn) btn.remove();
|
if (!onArtist) {
|
||||||
|
if (btn) btn.remove();
|
||||||
|
currentProbe = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// On artist pages, ask the backend what state the URL is in BEFORE
|
||||||
|
// injecting the button — so the chip can render the right state on
|
||||||
|
// first paint instead of flashing the generic "Add" copy and
|
||||||
|
// updating afterwards.
|
||||||
|
let probe;
|
||||||
|
try {
|
||||||
|
probe = await browser.runtime.sendMessage({ type: 'PROBE_SOURCE', url });
|
||||||
|
} catch (e) {
|
||||||
|
probe = { error: e?.message || 'probe failed' };
|
||||||
|
}
|
||||||
|
currentProbe = probe;
|
||||||
|
if (probe?.state === 'unknown_platform') {
|
||||||
|
if (btn) btn.remove();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
renderButton(probe);
|
||||||
}
|
}
|
||||||
|
|
||||||
function injectButton() {
|
function renderButton(probe) {
|
||||||
const btn = document.createElement('button');
|
let btn = document.getElementById('fc-add-source-btn');
|
||||||
btn.id = 'fc-add-source-btn';
|
if (!btn) {
|
||||||
|
btn = document.createElement('button');
|
||||||
|
btn.id = 'fc-add-source-btn';
|
||||||
|
btn.addEventListener('click', onClick);
|
||||||
|
document.body.appendChild(btn);
|
||||||
|
}
|
||||||
|
// Reset state classes so re-renders (SPA navigation) don't stack.
|
||||||
btn.className = 'fc-add-source-btn';
|
btn.className = 'fc-add-source-btn';
|
||||||
btn.textContent = '+ Add to FabledCurator';
|
btn.classList.add(`fc-add-source-btn--${stateModifier(probe)}`);
|
||||||
btn.addEventListener('click', onClick);
|
btn.textContent = labelFor(probe);
|
||||||
document.body.appendChild(btn);
|
btn.disabled = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function stateModifier(probe) {
|
||||||
|
if (!probe || probe.error) return 'new';
|
||||||
|
return ({
|
||||||
|
source_match: 'source-match',
|
||||||
|
artist_match: 'artist-match',
|
||||||
|
new: 'new',
|
||||||
|
})[probe.state] || 'new';
|
||||||
|
}
|
||||||
|
|
||||||
|
function labelFor(probe) {
|
||||||
|
if (!probe || probe.error) return '+ Add to FabledCurator';
|
||||||
|
const platformName = platformDisplayName(probe.platform);
|
||||||
|
const artistName = probe.artist?.name;
|
||||||
|
switch (probe.state) {
|
||||||
|
case 'source_match':
|
||||||
|
return `✓ In FabledCurator · ${platformName}`;
|
||||||
|
case 'artist_match':
|
||||||
|
return `+ Add ${platformName} source to ${artistName || 'artist'}`;
|
||||||
|
case 'new':
|
||||||
|
default:
|
||||||
|
return '+ Add to FabledCurator';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function platformDisplayName(key) {
|
||||||
|
return PLATFORMS[key]?.name || key || '';
|
||||||
}
|
}
|
||||||
|
|
||||||
async function onClick() {
|
async function onClick() {
|
||||||
const btn = document.getElementById('fc-add-source-btn');
|
const btn = document.getElementById('fc-add-source-btn');
|
||||||
|
if (!btn) return;
|
||||||
btn.disabled = true;
|
btn.disabled = true;
|
||||||
const original = btn.textContent;
|
const original = btn.textContent;
|
||||||
|
const probe = currentProbe;
|
||||||
|
|
||||||
|
if (probe?.state === 'source_match') {
|
||||||
|
btn.textContent = 'Opening…';
|
||||||
|
try {
|
||||||
|
const r = await browser.runtime.sendMessage({
|
||||||
|
type: 'OPEN_ARTIST_PAGE',
|
||||||
|
slug: probe.artist?.slug,
|
||||||
|
});
|
||||||
|
if (r?.error) showToast(`Error: ${r.error}`, 'error');
|
||||||
|
} catch (e) {
|
||||||
|
showToast(`Error: ${e.message}`, 'error');
|
||||||
|
} finally {
|
||||||
|
btn.disabled = false;
|
||||||
|
btn.textContent = original;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
btn.textContent = 'Adding…';
|
btn.textContent = 'Adding…';
|
||||||
try {
|
try {
|
||||||
const r = await browser.runtime.sendMessage({
|
const r = await browser.runtime.sendMessage({
|
||||||
type: 'ADD_AS_SOURCE',
|
type: 'ADD_AS_SOURCE',
|
||||||
url: window.location.href,
|
url: window.location.href,
|
||||||
});
|
});
|
||||||
if (r.error) {
|
if (r?.error) {
|
||||||
showToast(`Error: ${r.error}`, 'error');
|
showToast(`Error: ${r.error}`, 'error');
|
||||||
} else {
|
} else {
|
||||||
const verb = r.created_source ? 'Added' : 'Already a source for';
|
const verb = r.created_source ? 'Added' : 'Already a source for';
|
||||||
showToast(`${verb} ${r.artist?.name || 'artist'} (${r.source?.platform || ''})`, 'success');
|
showToast(`${verb} ${r.artist?.name || 'artist'} (${r.source?.platform || ''})`, 'success');
|
||||||
|
// Re-probe so the chip flips green without waiting for the next
|
||||||
|
// navigation.
|
||||||
|
evaluate();
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
showToast(`Error: ${e.message}`, 'error');
|
showToast(`Error: ${e.message}`, 'error');
|
||||||
|
|||||||
@@ -83,6 +83,12 @@ class FabledCuratorAPI {
|
|||||||
quickAddSource(url) {
|
quickAddSource(url) {
|
||||||
return this.request('POST', '/extension/quick-add-source', { url });
|
return this.request('POST', '/extension/quick-add-source', { url });
|
||||||
}
|
}
|
||||||
|
probeSource(url) {
|
||||||
|
// Read-only existence check. Drives the content-script chip's
|
||||||
|
// color/copy BEFORE the operator clicks Add.
|
||||||
|
const qs = new URLSearchParams({ url }).toString();
|
||||||
|
return this.request('GET', `/extension/probe?${qs}`);
|
||||||
|
}
|
||||||
|
|
||||||
// Connection test = the cheapest read with auth.
|
// Connection test = the cheapest read with auth.
|
||||||
testConnection() {
|
testConnection() {
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"manifest_version": 3,
|
"manifest_version": 3,
|
||||||
"name": "FabledCurator",
|
"name": "FabledCurator",
|
||||||
"version": "1.0.5",
|
"version": "1.0.6",
|
||||||
"description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.",
|
"description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.",
|
||||||
|
|
||||||
"browser_specific_settings": {
|
"browser_specific_settings": {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "fabledcurator-extension",
|
"name": "fabledcurator-extension",
|
||||||
"version": "1.0.5",
|
"version": "1.0.6",
|
||||||
"private": true,
|
"private": true,
|
||||||
"description": "Firefox extension for FabledCurator",
|
"description": "Firefox extension for FabledCurator",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
@@ -26,6 +26,8 @@
|
|||||||
"vue-tsc": "^2.0.0",
|
"vue-tsc": "^2.0.0",
|
||||||
"vite-plugin-vuetify": "^2.0.0",
|
"vite-plugin-vuetify": "^2.0.0",
|
||||||
"sass": "^1.71.0",
|
"sass": "^1.71.0",
|
||||||
"vitest": "^2.1.0"
|
"vitest": "^2.1.0",
|
||||||
|
"@vue/test-utils": "^2.4.0",
|
||||||
|
"happy-dom": "^15.0.0"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,146 @@
|
|||||||
|
<template>
|
||||||
|
<v-menu location="bottom start" :close-on-content-click="false">
|
||||||
|
<template #activator="{ props }">
|
||||||
|
<button
|
||||||
|
v-bind="props" type="button"
|
||||||
|
class="fc-pulse" :class="`fc-pulse--${schedHealth}`"
|
||||||
|
:aria-label="`pipeline: ${running} running, ${queued} queued, ${failing} failing`"
|
||||||
|
>
|
||||||
|
<span class="fc-pulse__dot" />
|
||||||
|
<span v-if="running" class="fc-pulse__stat">
|
||||||
|
<v-icon size="13">mdi-progress-download</v-icon>{{ running }}
|
||||||
|
</span>
|
||||||
|
<span v-if="queued" class="fc-pulse__stat">
|
||||||
|
<v-icon size="13">mdi-tray-full</v-icon>{{ queued }}
|
||||||
|
</span>
|
||||||
|
<span v-if="failing" class="fc-pulse__stat fc-pulse__stat--err">
|
||||||
|
<v-icon size="13">mdi-alert-circle</v-icon>{{ failing }}
|
||||||
|
</span>
|
||||||
|
<span v-if="!running && !queued && !failing" class="fc-pulse__idle">idle</span>
|
||||||
|
</button>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<v-card min-width="300" class="fc-pulse__panel pa-3">
|
||||||
|
<div class="fc-pulse__row">
|
||||||
|
<span class="fc-pulse__rowdot" :class="`fc-pulse--${schedHealth}`" />
|
||||||
|
<strong>Scheduler</strong>
|
||||||
|
<v-spacer />
|
||||||
|
<span class="fc-pulse__muted">{{ schedLabel }}</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="fc-pulse__sec">
|
||||||
|
<div class="fc-pulse__sechead">Queues</div>
|
||||||
|
<div v-if="busyQueues.length === 0" class="fc-pulse__muted">All idle</div>
|
||||||
|
<div v-for="q in busyQueues" :key="q.name" class="fc-pulse__qrow">
|
||||||
|
<span>{{ q.name }}</span><v-spacer /><strong>{{ q.depth }}</strong>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="fc-pulse__sec fc-pulse__grid">
|
||||||
|
<div><span class="fc-pulse__muted">Running</span><div class="fc-pulse__big">{{ running }}</div></div>
|
||||||
|
<div><span class="fc-pulse__muted">Queued</span><div class="fc-pulse__big">{{ queued }}</div></div>
|
||||||
|
<div>
|
||||||
|
<span class="fc-pulse__muted">Failures 24h</span>
|
||||||
|
<div class="fc-pulse__big" :class="{ 'fc-pulse__big--err': failing }">{{ failing }}</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<RouterLink
|
||||||
|
class="fc-pulse__link"
|
||||||
|
:to="{ path: '/subscriptions', query: { tab: 'downloads' } }"
|
||||||
|
>Open downloads →</RouterLink>
|
||||||
|
</v-card>
|
||||||
|
</v-menu>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed, onMounted, onUnmounted } from 'vue'
|
||||||
|
|
||||||
|
import { useSystemActivityStore } from '../stores/systemActivity.js'
|
||||||
|
import { formatRelative } from '../utils/date.js'
|
||||||
|
|
||||||
|
const store = useSystemActivityStore()
|
||||||
|
|
||||||
|
// Beat ticks every 60s; flag the scheduler stale past ~3 min.
|
||||||
|
const STALE_MS = 180_000
|
||||||
|
const summary = computed(() => store.summary)
|
||||||
|
const running = computed(() => summary.value?.running ?? 0)
|
||||||
|
const queued = computed(() => summary.value?.queued_total ?? 0)
|
||||||
|
const failing = computed(() => summary.value?.failing ?? 0)
|
||||||
|
|
||||||
|
const schedHealth = computed(() => {
|
||||||
|
const t = summary.value?.scheduler?.last_tick_at
|
||||||
|
if (!t) return 'unknown'
|
||||||
|
return (Date.now() - new Date(t).getTime()) <= STALE_MS ? 'ok' : 'stale'
|
||||||
|
})
|
||||||
|
const schedLabel = computed(() => {
|
||||||
|
const s = summary.value?.scheduler
|
||||||
|
if (!s) return '—'
|
||||||
|
const ran = formatRelative(s.last_tick_at, { nullText: 'never' })
|
||||||
|
if (s.due_now > 0) return `ran ${ran} · ${s.due_now} due`
|
||||||
|
return `ran ${ran}`
|
||||||
|
})
|
||||||
|
const busyQueues = computed(() => {
|
||||||
|
const q = summary.value?.queues || {}
|
||||||
|
return Object.entries(q)
|
||||||
|
.filter(([, depth]) => typeof depth === 'number' && depth > 0)
|
||||||
|
.map(([name, depth]) => ({ name, depth }))
|
||||||
|
})
|
||||||
|
|
||||||
|
const POLL_MS = 8000
|
||||||
|
let timer = null
|
||||||
|
onMounted(() => {
|
||||||
|
store.loadSummary()
|
||||||
|
timer = setInterval(() => {
|
||||||
|
if (!document.hidden) store.loadSummary()
|
||||||
|
}, POLL_MS)
|
||||||
|
})
|
||||||
|
onUnmounted(() => { if (timer) clearInterval(timer) })
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-pulse {
|
||||||
|
display: inline-flex; align-items: center; gap: 6px;
|
||||||
|
padding: 3px 8px; border-radius: 999px;
|
||||||
|
background: rgb(var(--v-theme-on-surface) / 0.08);
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
font-size: 0.78rem; font-variant-numeric: tabular-nums;
|
||||||
|
cursor: pointer; border: 0;
|
||||||
|
}
|
||||||
|
.fc-pulse:hover { background: rgb(var(--v-theme-on-surface) / 0.16); }
|
||||||
|
.fc-pulse__dot {
|
||||||
|
width: 8px; height: 8px; border-radius: 50%; flex: 0 0 auto;
|
||||||
|
background: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
}
|
||||||
|
.fc-pulse--ok .fc-pulse__dot,
|
||||||
|
.fc-pulse--ok.fc-pulse__rowdot { background: rgb(var(--v-theme-success)); }
|
||||||
|
.fc-pulse--stale .fc-pulse__dot,
|
||||||
|
.fc-pulse--stale.fc-pulse__rowdot { background: rgb(var(--v-theme-error)); }
|
||||||
|
.fc-pulse__stat { display: inline-flex; align-items: center; gap: 2px; }
|
||||||
|
.fc-pulse__stat--err { color: rgb(var(--v-theme-error)); }
|
||||||
|
.fc-pulse__idle {
|
||||||
|
text-transform: uppercase; letter-spacing: 0.06em;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant)); font-size: 0.68rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.fc-pulse__panel { background: rgb(var(--v-theme-surface)); }
|
||||||
|
.fc-pulse__row { display: flex; align-items: center; gap: 8px; }
|
||||||
|
.fc-pulse__rowdot {
|
||||||
|
width: 9px; height: 9px; border-radius: 50%;
|
||||||
|
background: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
}
|
||||||
|
.fc-pulse__muted { color: rgb(var(--v-theme-on-surface-variant)); font-size: 0.8rem; }
|
||||||
|
.fc-pulse__sec { margin-top: 12px; }
|
||||||
|
.fc-pulse__sechead {
|
||||||
|
font-size: 0.72rem; text-transform: uppercase; letter-spacing: 0.05em;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant)); margin-bottom: 4px;
|
||||||
|
}
|
||||||
|
.fc-pulse__qrow { display: flex; align-items: center; font-size: 0.85rem; padding: 2px 0; }
|
||||||
|
.fc-pulse__grid { display: flex; gap: 16px; }
|
||||||
|
.fc-pulse__big { font-size: 1.3rem; font-weight: 700; font-variant-numeric: tabular-nums; }
|
||||||
|
.fc-pulse__big--err { color: rgb(var(--v-theme-error)); }
|
||||||
|
.fc-pulse__link {
|
||||||
|
display: inline-block; margin-top: 14px;
|
||||||
|
color: rgb(var(--v-theme-accent)); text-decoration: none; font-size: 0.85rem;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -8,6 +8,7 @@
|
|||||||
<span class="fc-health" :title="health.label">
|
<span class="fc-health" :title="health.label">
|
||||||
<v-icon size="x-small" :color="health.color">{{ health.icon }}</v-icon>
|
<v-icon size="x-small" :color="health.color">{{ health.icon }}</v-icon>
|
||||||
</span>
|
</span>
|
||||||
|
<PipelineStatusChip />
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<nav class="fc-links">
|
<nav class="fc-links">
|
||||||
@@ -29,6 +30,7 @@
|
|||||||
import { computed, onMounted } from 'vue'
|
import { computed, onMounted } from 'vue'
|
||||||
import router, { FRONT_DOOR } from '../router.js'
|
import router, { FRONT_DOOR } from '../router.js'
|
||||||
import { useSystemStore } from '../stores/system.js'
|
import { useSystemStore } from '../stores/system.js'
|
||||||
|
import PipelineStatusChip from './PipelineStatusChip.vue'
|
||||||
|
|
||||||
const system = useSystemStore()
|
const system = useSystemStore()
|
||||||
onMounted(() => system.refreshHealth())
|
onMounted(() => system.refreshHealth())
|
||||||
|
|||||||
@@ -35,6 +35,7 @@
|
|||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed } from 'vue'
|
import { computed } from 'vue'
|
||||||
|
import { formatLocalDate } from '../../utils/date.js'
|
||||||
|
|
||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
name: { type: String, required: true },
|
name: { type: String, required: true },
|
||||||
@@ -52,7 +53,7 @@ const stats = computed(() => {
|
|||||||
parts.push(`${props.imageCount} image${props.imageCount === 1 ? '' : 's'}`)
|
parts.push(`${props.imageCount} image${props.imageCount === 1 ? '' : 's'}`)
|
||||||
}
|
}
|
||||||
if (props.lastAdded) {
|
if (props.lastAdded) {
|
||||||
parts.push(`last added ${props.lastAdded.slice(0, 10)}`)
|
parts.push(`last added ${formatLocalDate(props.lastAdded)}`)
|
||||||
}
|
}
|
||||||
return parts.join(' · ')
|
return parts.join(' · ')
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -28,10 +28,11 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { onMounted, onUnmounted, ref, watch } from 'vue'
|
import { onMounted, ref, watch } from 'vue'
|
||||||
import { RouterLink } from 'vue-router'
|
import { RouterLink } from 'vue-router'
|
||||||
|
|
||||||
import { usePostsStore } from '../../stores/posts.js'
|
import { usePostsStore } from '../../stores/posts.js'
|
||||||
|
import { useInfiniteScroll } from '../../composables/useInfiniteScroll.js'
|
||||||
import PostCard from '../posts/PostCard.vue'
|
import PostCard from '../posts/PostCard.vue'
|
||||||
|
|
||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
@@ -42,7 +43,6 @@ defineEmits(['switch-tab'])
|
|||||||
|
|
||||||
const store = usePostsStore()
|
const store = usePostsStore()
|
||||||
const sentinel = ref(null)
|
const sentinel = ref(null)
|
||||||
let observer = null
|
|
||||||
|
|
||||||
async function reload () {
|
async function reload () {
|
||||||
await store.loadInitial({ artist_id: props.artistId, platform: null })
|
await store.loadInitial({ artist_id: props.artistId, platform: null })
|
||||||
@@ -50,19 +50,9 @@ async function reload () {
|
|||||||
|
|
||||||
watch(() => props.artistId, reload)
|
watch(() => props.artistId, reload)
|
||||||
|
|
||||||
onMounted(async () => {
|
useInfiniteScroll(sentinel, () => store.loadMore(), { rootMargin: '400px 0px' })
|
||||||
await reload()
|
|
||||||
observer = new IntersectionObserver((entries) => {
|
|
||||||
if (entries.some(e => e.isIntersecting)) {
|
|
||||||
store.loadMore()
|
|
||||||
}
|
|
||||||
}, { rootMargin: '400px 0px' })
|
|
||||||
if (sentinel.value) observer.observe(sentinel.value)
|
|
||||||
})
|
|
||||||
|
|
||||||
onUnmounted(() => {
|
onMounted(reload)
|
||||||
if (observer) observer.disconnect()
|
|
||||||
})
|
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
|||||||
@@ -62,6 +62,7 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
import { onMounted, ref } from 'vue'
|
import { onMounted, ref } from 'vue'
|
||||||
|
|
||||||
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||||
@@ -94,7 +95,7 @@ async function onPreview() {
|
|||||||
try {
|
try {
|
||||||
preview.value = await store.previewMinDim(minW.value, minH.value)
|
preview.value = await store.previewMinDim(minW.value, minH.value)
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
window.__fcToast?.({ text: `Preview failed: ${e.message}`, type: 'error' })
|
toast({ text: `Preview failed: ${e.message}`, type: 'error' })
|
||||||
} finally {
|
} finally {
|
||||||
busy.value = false
|
busy.value = false
|
||||||
}
|
}
|
||||||
@@ -108,12 +109,12 @@ function onDeleteClick() {
|
|||||||
async function onConfirmedDelete(token) {
|
async function onConfirmedDelete(token) {
|
||||||
try {
|
try {
|
||||||
const res = await store.deleteMinDim(minW.value, minH.value, token)
|
const res = await store.deleteMinDim(minW.value, minH.value, token)
|
||||||
window.__fcToast?.({
|
toast({
|
||||||
text: `Deleted ${res.deleted} image(s)`, type: 'success',
|
text: `Deleted ${res.deleted} image(s)`, type: 'success',
|
||||||
})
|
})
|
||||||
preview.value = null
|
preview.value = null
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
window.__fcToast?.({ text: `Delete failed: ${e.message}`, type: 'error' })
|
toast({ text: `Delete failed: ${e.message}`, type: 'error' })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -93,6 +93,7 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
import { onMounted, onUnmounted, ref } from 'vue'
|
import { onMounted, onUnmounted, ref } from 'vue'
|
||||||
|
|
||||||
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||||
@@ -124,7 +125,7 @@ function startPoll(id) {
|
|||||||
if (fresh.status !== 'running') stopPoll()
|
if (fresh.status !== 'running') stopPoll()
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
stopPoll()
|
stopPoll()
|
||||||
window.__fcToast?.({ text: `Audit poll failed: ${e.message}`, type: 'error' })
|
toast({ text: `Audit poll failed: ${e.message}`, type: 'error' })
|
||||||
}
|
}
|
||||||
}, 5000)
|
}, 5000)
|
||||||
}
|
}
|
||||||
@@ -142,7 +143,7 @@ async function onStart() {
|
|||||||
audit.value = await store.getAudit(res.audit_id)
|
audit.value = await store.getAudit(res.audit_id)
|
||||||
startPoll(res.audit_id)
|
startPoll(res.audit_id)
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
window.__fcToast?.({ text: `Scan start failed: ${e.message}`, type: 'error' })
|
toast({ text: `Scan start failed: ${e.message}`, type: 'error' })
|
||||||
} finally {
|
} finally {
|
||||||
busy.value = false
|
busy.value = false
|
||||||
}
|
}
|
||||||
@@ -155,7 +156,7 @@ async function onCancel() {
|
|||||||
audit.value = await store.getAudit(audit.value.id)
|
audit.value = await store.getAudit(audit.value.id)
|
||||||
stopPoll()
|
stopPoll()
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
window.__fcToast?.({ text: `Cancel failed: ${e.message}`, type: 'error' })
|
toast({ text: `Cancel failed: ${e.message}`, type: 'error' })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -167,12 +168,12 @@ function onApplyClick() {
|
|||||||
async function onConfirmedApply(token) {
|
async function onConfirmedApply(token) {
|
||||||
try {
|
try {
|
||||||
const res = await store.applyAudit(audit.value.id, token)
|
const res = await store.applyAudit(audit.value.id, token)
|
||||||
window.__fcToast?.({
|
toast({
|
||||||
text: `Deleted ${res.deleted} image(s)`, type: 'success',
|
text: `Deleted ${res.deleted} image(s)`, type: 'success',
|
||||||
})
|
})
|
||||||
audit.value = await store.getAudit(audit.value.id)
|
audit.value = await store.getAudit(audit.value.id)
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
window.__fcToast?.({ text: `Apply failed: ${e.message}`, type: 'error' })
|
toast({ text: `Apply failed: ${e.message}`, type: 'error' })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -80,6 +80,7 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
import { onMounted, onUnmounted, ref } from 'vue'
|
import { onMounted, onUnmounted, ref } from 'vue'
|
||||||
|
|
||||||
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||||
@@ -109,7 +110,7 @@ function startPoll(id) {
|
|||||||
if (fresh.status !== 'running') stopPoll()
|
if (fresh.status !== 'running') stopPoll()
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
stopPoll()
|
stopPoll()
|
||||||
window.__fcToast?.({ text: `Audit poll failed: ${e.message}`, type: 'error' })
|
toast({ text: `Audit poll failed: ${e.message}`, type: 'error' })
|
||||||
}
|
}
|
||||||
}, 5000)
|
}, 5000)
|
||||||
}
|
}
|
||||||
@@ -125,7 +126,7 @@ async function onStart() {
|
|||||||
audit.value = await store.getAudit(res.audit_id)
|
audit.value = await store.getAudit(res.audit_id)
|
||||||
startPoll(res.audit_id)
|
startPoll(res.audit_id)
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
window.__fcToast?.({ text: `Scan start failed: ${e.message}`, type: 'error' })
|
toast({ text: `Scan start failed: ${e.message}`, type: 'error' })
|
||||||
} finally {
|
} finally {
|
||||||
busy.value = false
|
busy.value = false
|
||||||
}
|
}
|
||||||
@@ -138,7 +139,7 @@ async function onCancel() {
|
|||||||
audit.value = await store.getAudit(audit.value.id)
|
audit.value = await store.getAudit(audit.value.id)
|
||||||
stopPoll()
|
stopPoll()
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
window.__fcToast?.({ text: `Cancel failed: ${e.message}`, type: 'error' })
|
toast({ text: `Cancel failed: ${e.message}`, type: 'error' })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -150,12 +151,12 @@ function onApplyClick() {
|
|||||||
async function onConfirmedApply(token) {
|
async function onConfirmedApply(token) {
|
||||||
try {
|
try {
|
||||||
const res = await store.applyAudit(audit.value.id, token)
|
const res = await store.applyAudit(audit.value.id, token)
|
||||||
window.__fcToast?.({
|
toast({
|
||||||
text: `Deleted ${res.deleted} image(s)`, type: 'success',
|
text: `Deleted ${res.deleted} image(s)`, type: 'success',
|
||||||
})
|
})
|
||||||
audit.value = await store.getAudit(audit.value.id)
|
audit.value = await store.getAudit(audit.value.id)
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
window.__fcToast?.({ text: `Apply failed: ${e.message}`, type: 'error' })
|
toast({ text: `Apply failed: ${e.message}`, type: 'error' })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -33,6 +33,7 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
import { computed, ref, watch } from 'vue'
|
import { computed, ref, watch } from 'vue'
|
||||||
|
|
||||||
import { copyText } from '../../utils/clipboard.js'
|
import { copyText } from '../../utils/clipboard.js'
|
||||||
@@ -119,7 +120,7 @@ async function onCopy () {
|
|||||||
if (copiedTimer) clearTimeout(copiedTimer)
|
if (copiedTimer) clearTimeout(copiedTimer)
|
||||||
copiedTimer = setTimeout(() => { copied.value = false }, 1500)
|
copiedTimer = setTimeout(() => { copied.value = false }, 1500)
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
window.__fcToast?.({ text: `Copy failed: ${e.message}`, type: 'error' })
|
toast({ text: `Copy failed: ${e.message}`, type: 'error' })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -26,6 +26,7 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
import { onMounted, ref } from 'vue'
|
import { onMounted, ref } from 'vue'
|
||||||
import { useCredentialsStore } from '../../stores/credentials.js'
|
import { useCredentialsStore } from '../../stores/credentials.js'
|
||||||
import { copyText } from '../../utils/clipboard.js'
|
import { copyText } from '../../utils/clipboard.js'
|
||||||
@@ -39,9 +40,9 @@ async function copyKey() {
|
|||||||
if (!store.extensionKey) return
|
if (!store.extensionKey) return
|
||||||
try {
|
try {
|
||||||
await copyText(store.extensionKey)
|
await copyText(store.extensionKey)
|
||||||
globalThis.window?.__fcToast?.({ text: 'Copied', type: 'success' })
|
toast({ text: 'Copied', type: 'success' })
|
||||||
} catch {
|
} catch {
|
||||||
globalThis.window?.__fcToast?.({ text: 'Copy failed', type: 'error' })
|
toast({ text: 'Copy failed', type: 'error' })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -50,7 +51,7 @@ function confirmRotate() { showRotateConfirm.value = true }
|
|||||||
async function doRotate() {
|
async function doRotate() {
|
||||||
showRotateConfirm.value = false
|
showRotateConfirm.value = false
|
||||||
await store.rotateKey()
|
await store.rotateKey()
|
||||||
globalThis.window?.__fcToast?.({ text: 'Key rotated', type: 'success' })
|
toast({ text: 'Key rotated', type: 'success' })
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|||||||
@@ -32,6 +32,8 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { formatLocalDate } from '../../utils/date.js'
|
||||||
|
|
||||||
defineProps({
|
defineProps({
|
||||||
platform: { type: Object, required: true },
|
platform: { type: Object, required: true },
|
||||||
credential: { type: Object, default: null },
|
credential: { type: Object, default: null },
|
||||||
@@ -39,8 +41,7 @@ defineProps({
|
|||||||
defineEmits(['replace', 'remove'])
|
defineEmits(['replace', 'remove'])
|
||||||
|
|
||||||
function fmtDate(iso) {
|
function fmtDate(iso) {
|
||||||
if (!iso) return '—'
|
return iso ? formatLocalDate(iso) : '—'
|
||||||
return iso.slice(0, 10)
|
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|||||||
@@ -30,8 +30,9 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { ref, computed, onMounted, onUnmounted, watch } from 'vue'
|
import { ref, computed } from 'vue'
|
||||||
import { usePolyMasonry } from '../../composables/usePolyMasonry.js'
|
import { usePolyMasonry } from '../../composables/usePolyMasonry.js'
|
||||||
|
import { useInfiniteScroll } from '../../composables/useInfiniteScroll.js'
|
||||||
|
|
||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
items: { type: Array, default: () => [] },
|
items: { type: Array, default: () => [] },
|
||||||
@@ -78,20 +79,18 @@ function aspectStyle(item) {
|
|||||||
return { aspectRatio: `${w} / ${h}` }
|
return { aspectRatio: `${w} / ${h}` }
|
||||||
}
|
}
|
||||||
|
|
||||||
let observer = null
|
// Larger rootMargin than the composable default (600px) because the
|
||||||
function attachObserver() {
|
// sentinel sits at the BOTTOM of the masonry container, whose height is
|
||||||
if (observer) observer.disconnect()
|
// the MAX of the column heights. A single tall image (long manga page,
|
||||||
if (!sentinelEl.value) return
|
// panorama) in one column pushes the sentinel way past the visible
|
||||||
observer = new IntersectionObserver(([entry]) => {
|
// bottom of the SHORTER columns — the user reads the short-column
|
||||||
if (entry.isIntersecting && props.hasMore && !props.loading) {
|
// bottoms long before the sentinel comes into view, and load-more
|
||||||
emit('load-more')
|
// fires too late. 2400px ≈ 2-3 screen-heights of pre-emptive trigger,
|
||||||
}
|
// comfortably covering typical tall-image heights. Operator-flagged
|
||||||
}, { rootMargin: '600px' })
|
// 2026-05-30.
|
||||||
observer.observe(sentinelEl.value)
|
useInfiniteScroll(sentinelEl, () => {
|
||||||
}
|
if (props.hasMore && !props.loading) emit('load-more')
|
||||||
watch(sentinelEl, attachObserver)
|
}, { rootMargin: '2400px' })
|
||||||
onMounted(attachObserver)
|
|
||||||
onUnmounted(() => observer && observer.disconnect())
|
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
@@ -100,32 +99,54 @@ onUnmounted(() => observer && observer.disconnect())
|
|||||||
.fc-masonry__item {
|
.fc-masonry__item {
|
||||||
display: block; padding: 0; border: 0; background: none;
|
display: block; padding: 0; border: 0; background: none;
|
||||||
cursor: pointer; width: 100%;
|
cursor: pointer; width: 100%;
|
||||||
|
overflow: hidden; border-radius: 4px;
|
||||||
}
|
}
|
||||||
.fc-masonry__item img {
|
.fc-masonry__item img {
|
||||||
width: 100%; height: auto; display: block; border-radius: 4px;
|
width: 100%; height: auto; display: block;
|
||||||
background: rgb(var(--v-theme-surface-light));
|
background: rgb(var(--v-theme-surface-light));
|
||||||
|
/* IR-parity hover: zoom + brighten the thumbnail (style.css ~1772). */
|
||||||
|
transition: transform 0.3s ease, filter 0.3s ease;
|
||||||
|
}
|
||||||
|
.fc-masonry__item:hover img {
|
||||||
|
transform: scale(1.03);
|
||||||
|
filter: brightness(1.1);
|
||||||
}
|
}
|
||||||
.fc-masonry__sentinel {
|
.fc-masonry__sentinel {
|
||||||
display: flex; justify-content: center; padding: 32px 0; min-height: 60px;
|
display: flex; justify-content: center; padding: 32px 0; min-height: 60px;
|
||||||
}
|
}
|
||||||
.fc-masonry__end { text-align: center; padding: 32px 0; }
|
.fc-masonry__end { text-align: center; padding: 32px 0; }
|
||||||
|
|
||||||
/* IR-parity stagger fade-in for showcase entry / shuffle. 60ms between
|
/* Cascade entry: each tile flips up out of a backward tilt and settles
|
||||||
items, 250ms each — matches IR's `itemFadeIn` keyframe (style.css
|
into place, one at a time — more pronounced than a plain fade so the
|
||||||
~line 1834). Honors prefers-reduced-motion. */
|
showcase reads as an "experience" (operator-flagged 2026-05-28). The
|
||||||
|
`both` fill holds the hidden/tilted 0% state until each tile's staggered
|
||||||
|
turn; the cubic-bezier overshoots slightly past flat then settles.
|
||||||
|
Honors prefers-reduced-motion. Tunables: tilt (-28deg), stagger (70ms),
|
||||||
|
duration (0.6s). */
|
||||||
@keyframes fc-masonry-item-in {
|
@keyframes fc-masonry-item-in {
|
||||||
from { opacity: 0; transform: translateY(12px); }
|
0% {
|
||||||
to { opacity: 1; transform: translateY(0); }
|
opacity: 0;
|
||||||
|
transform: perspective(1000px) rotateX(-28deg) translateY(26px) scale(0.95);
|
||||||
|
}
|
||||||
|
55% { opacity: 1; }
|
||||||
|
100% {
|
||||||
|
opacity: 1;
|
||||||
|
transform: perspective(1000px) rotateX(0deg) translateY(0) scale(1);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
.fc-masonry__item--anim {
|
.fc-masonry__item--anim {
|
||||||
animation: fc-masonry-item-in 0.25s ease forwards;
|
transform-origin: center top;
|
||||||
animation-delay: calc(var(--stagger-index, 0) * 60ms);
|
backface-visibility: hidden;
|
||||||
opacity: 0;
|
animation: fc-masonry-item-in 0.6s cubic-bezier(0.34, 1.45, 0.64, 1) both;
|
||||||
|
animation-delay: calc(var(--stagger-index, 0) * 70ms);
|
||||||
}
|
}
|
||||||
@media (prefers-reduced-motion: reduce) {
|
@media (prefers-reduced-motion: reduce) {
|
||||||
.fc-masonry__item--anim {
|
.fc-masonry__item--anim {
|
||||||
animation: none;
|
animation: none;
|
||||||
opacity: 1;
|
opacity: 1;
|
||||||
|
transform: none;
|
||||||
}
|
}
|
||||||
|
.fc-masonry__item img { transition: none; }
|
||||||
|
.fc-masonry__item:hover img { transform: none; }
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -7,7 +7,7 @@
|
|||||||
</v-card-title>
|
</v-card-title>
|
||||||
<v-card-text>
|
<v-card-text>
|
||||||
<p class="text-caption" style="opacity: 0.75">
|
<p class="text-caption" style="opacity: 0.75">
|
||||||
{{ event.started_at }} → {{ event.finished_at || '(running)' }}
|
{{ formatDateTime(event.started_at) }} → {{ event.finished_at ? formatDateTime(event.finished_at) : '(running)' }}
|
||||||
({{ fmtDuration(event.summary?.duration_seconds) }})
|
({{ fmtDuration(event.summary?.duration_seconds) }})
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
@@ -35,20 +35,53 @@
|
|||||||
</ul>
|
</ul>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
|
<template v-if="event.error">
|
||||||
|
<div class="fc-dl-blockhead mt-4">
|
||||||
|
<h3 class="text-subtitle-2">Error</h3>
|
||||||
|
<v-btn
|
||||||
|
size="x-small" variant="text" prepend-icon="mdi-content-copy"
|
||||||
|
@click="onCopy('Error', event.error)"
|
||||||
|
>Copy</v-btn>
|
||||||
|
</div>
|
||||||
|
<pre class="fc-dl-pre">{{ event.error }}</pre>
|
||||||
|
</template>
|
||||||
|
|
||||||
<template v-if="errorsWarnings">
|
<template v-if="errorsWarnings">
|
||||||
<h3 class="text-subtitle-2 mt-4">Errors & warnings</h3>
|
<div class="fc-dl-blockhead mt-4">
|
||||||
|
<h3 class="text-subtitle-2">Errors & warnings</h3>
|
||||||
|
<v-btn
|
||||||
|
size="x-small" variant="text" prepend-icon="mdi-content-copy"
|
||||||
|
@click="onCopy('Errors & warnings', errorsWarnings)"
|
||||||
|
>Copy</v-btn>
|
||||||
|
</div>
|
||||||
<pre class="fc-dl-pre">{{ errorsWarnings }}</pre>
|
<pre class="fc-dl-pre">{{ errorsWarnings }}</pre>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<v-expansion-panels class="mt-4">
|
<v-expansion-panels class="mt-4">
|
||||||
<v-expansion-panel>
|
<v-expansion-panel>
|
||||||
<v-expansion-panel-title>Raw stdout</v-expansion-panel-title>
|
<v-expansion-panel-title>
|
||||||
|
<span>Raw stdout</span>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn
|
||||||
|
size="x-small" variant="text" prepend-icon="mdi-content-copy"
|
||||||
|
class="me-2"
|
||||||
|
@click.stop="onCopy('stdout', event.metadata?.stdout || '')"
|
||||||
|
>Copy</v-btn>
|
||||||
|
</v-expansion-panel-title>
|
||||||
<v-expansion-panel-text>
|
<v-expansion-panel-text>
|
||||||
<pre class="fc-dl-pre">{{ event.metadata?.stdout || '(empty)' }}</pre>
|
<pre class="fc-dl-pre">{{ event.metadata?.stdout || '(empty)' }}</pre>
|
||||||
</v-expansion-panel-text>
|
</v-expansion-panel-text>
|
||||||
</v-expansion-panel>
|
</v-expansion-panel>
|
||||||
<v-expansion-panel>
|
<v-expansion-panel>
|
||||||
<v-expansion-panel-title>Raw stderr</v-expansion-panel-title>
|
<v-expansion-panel-title>
|
||||||
|
<span>Raw stderr</span>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn
|
||||||
|
size="x-small" variant="text" prepend-icon="mdi-content-copy"
|
||||||
|
class="me-2"
|
||||||
|
@click.stop="onCopy('stderr', event.metadata?.stderr || '')"
|
||||||
|
>Copy</v-btn>
|
||||||
|
</v-expansion-panel-title>
|
||||||
<v-expansion-panel-text>
|
<v-expansion-panel-text>
|
||||||
<pre class="fc-dl-pre">{{ event.metadata?.stderr || '(empty)' }}</pre>
|
<pre class="fc-dl-pre">{{ event.metadata?.stderr || '(empty)' }}</pre>
|
||||||
</v-expansion-panel-text>
|
</v-expansion-panel-text>
|
||||||
@@ -56,6 +89,10 @@
|
|||||||
</v-expansion-panels>
|
</v-expansion-panels>
|
||||||
</v-card-text>
|
</v-card-text>
|
||||||
<v-card-actions>
|
<v-card-actions>
|
||||||
|
<v-btn
|
||||||
|
variant="text" prepend-icon="mdi-content-copy"
|
||||||
|
@click="onCopy('All diagnostics', allDiagnostics)"
|
||||||
|
>Copy all diagnostics</v-btn>
|
||||||
<v-spacer />
|
<v-spacer />
|
||||||
<v-btn variant="text" @click="onClose(false)">Close</v-btn>
|
<v-btn variant="text" @click="onClose(false)">Close</v-btn>
|
||||||
</v-card-actions>
|
</v-card-actions>
|
||||||
@@ -64,8 +101,12 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
import { computed } from 'vue'
|
import { computed } from 'vue'
|
||||||
|
|
||||||
|
import { copyText } from '../../utils/clipboard.js'
|
||||||
|
import { formatDateTime } from '../../utils/date.js'
|
||||||
|
|
||||||
const props = defineProps({ event: { type: Object, default: null } })
|
const props = defineProps({ event: { type: Object, default: null } })
|
||||||
const emit = defineEmits(['close'])
|
const emit = defineEmits(['close'])
|
||||||
|
|
||||||
@@ -74,6 +115,32 @@ const summary = computed(() => props.event?.metadata?.import_summary || {})
|
|||||||
const quarantinedPaths = computed(() => props.event?.metadata?.quarantined_paths || [])
|
const quarantinedPaths = computed(() => props.event?.metadata?.quarantined_paths || [])
|
||||||
const errorsWarnings = computed(() => props.event?.metadata?.stderr_errors_warnings || '')
|
const errorsWarnings = computed(() => props.event?.metadata?.stderr_errors_warnings || '')
|
||||||
|
|
||||||
|
// One combined block for "research the issue elsewhere" — header line +
|
||||||
|
// error + full stdout/stderr. Built lazily from the current event.
|
||||||
|
const allDiagnostics = computed(() => {
|
||||||
|
const e = props.event
|
||||||
|
if (!e) return ''
|
||||||
|
const md = e.metadata || {}
|
||||||
|
return [
|
||||||
|
`event #${e.id} · ${e.platform || '—'} · ${e.artist_name || '—'}`,
|
||||||
|
`status: ${e.status}`,
|
||||||
|
`started: ${e.started_at} finished: ${e.finished_at || '(running)'}`,
|
||||||
|
e.error ? `\n--- error ---\n${e.error}` : '',
|
||||||
|
errorsWarnings.value ? `\n--- errors & warnings ---\n${errorsWarnings.value}` : '',
|
||||||
|
`\n--- stdout ---\n${md.stdout || '(empty)'}`,
|
||||||
|
`\n--- stderr ---\n${md.stderr || '(empty)'}`,
|
||||||
|
].filter(Boolean).join('\n')
|
||||||
|
})
|
||||||
|
|
||||||
|
async function onCopy(label, text) {
|
||||||
|
try {
|
||||||
|
await copyText(text || '')
|
||||||
|
toast({ text: `${label} copied`, type: 'success' })
|
||||||
|
} catch (e) {
|
||||||
|
toast({ text: `Copy failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const statusColor = computed(() => ({
|
const statusColor = computed(() => ({
|
||||||
ok: 'success', error: 'error', running: 'info',
|
ok: 'success', error: 'error', running: 'info',
|
||||||
pending: 'secondary', skipped: 'warning',
|
pending: 'secondary', skipped: 'warning',
|
||||||
@@ -114,4 +181,8 @@ function onClose() {
|
|||||||
word-break: break-all;
|
word-break: break-all;
|
||||||
}
|
}
|
||||||
.fc-dl-quar { padding-left: 1.5rem; font-size: 0.85rem; }
|
.fc-dl-quar { padding-left: 1.5rem; font-size: 0.85rem; }
|
||||||
|
.fc-dl-blockhead {
|
||||||
|
display: flex; align-items: center; justify-content: space-between;
|
||||||
|
gap: 0.5rem;
|
||||||
|
}
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -1,47 +1,112 @@
|
|||||||
<template>
|
<template>
|
||||||
<div class="fc-dl-row" @click="$emit('open', event.id)">
|
<div
|
||||||
<v-icon :icon="statusIcon" :color="statusColor" size="small" />
|
class="fc-dl-row"
|
||||||
|
:class="[`fc-dl-row--${event.status || 'unknown'}`]"
|
||||||
|
@click="$emit('open', event.id)"
|
||||||
|
>
|
||||||
|
<!-- Colored left edge marks the run's status; matches the row's
|
||||||
|
status-chip color but reads at a glance without needing to
|
||||||
|
parse the chip text. -->
|
||||||
|
<div class="fc-dl-row__bar" />
|
||||||
|
|
||||||
|
<v-chip
|
||||||
|
:color="statusColor"
|
||||||
|
size="small"
|
||||||
|
variant="tonal"
|
||||||
|
:prepend-icon="statusIcon"
|
||||||
|
class="fc-dl-row__status"
|
||||||
|
>{{ statusLabel }}</v-chip>
|
||||||
|
|
||||||
<RouterLink
|
<RouterLink
|
||||||
v-if="event.artist_slug"
|
v-if="event.artist_slug"
|
||||||
:to="`/artist/${event.artist_slug}`"
|
:to="`/artist/${event.artist_slug}`"
|
||||||
class="fc-dl-row__artist"
|
class="fc-dl-row__artist"
|
||||||
@click.stop
|
@click.stop
|
||||||
>{{ event.artist_name }}</RouterLink>
|
>{{ event.artist_name }}</RouterLink>
|
||||||
<span v-else class="fc-dl-row__artist">—</span>
|
<span v-else class="fc-dl-row__artist fc-dl-row__artist--missing">—</span>
|
||||||
<v-chip size="x-small" variant="tonal">{{ event.platform || '—' }}</v-chip>
|
|
||||||
<span class="fc-dl-row__time">{{ fmtTime(event.started_at) }}</span>
|
<PlatformChip
|
||||||
<span class="fc-dl-row__files">{{ event.files_count }} files</span>
|
v-if="event.platform"
|
||||||
<span class="fc-dl-row__duration">{{ fmtDuration(event.summary?.duration_seconds) }}</span>
|
:platform="event.platform"
|
||||||
<span v-if="event.error" class="fc-dl-row__error">{{ event.error }}</span>
|
size="x-small"
|
||||||
|
class="fc-dl-row__platform"
|
||||||
|
/>
|
||||||
|
<span v-else class="fc-dl-row__platform-missing">—</span>
|
||||||
|
|
||||||
|
<span class="fc-dl-row__time" :title="event.started_at">
|
||||||
|
{{ fmtTime(event.started_at) }}
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<v-chip
|
||||||
|
v-if="event.files_count > 0"
|
||||||
|
size="x-small" variant="tonal" color="info"
|
||||||
|
prepend-icon="mdi-image-multiple"
|
||||||
|
class="fc-dl-row__files"
|
||||||
|
>{{ event.files_count }}</v-chip>
|
||||||
|
<span v-else class="fc-dl-row__no-files" aria-label="no new files">·</span>
|
||||||
|
|
||||||
|
<span class="fc-dl-row__duration">
|
||||||
|
{{ fmtDuration(event.summary?.duration_seconds) }}
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<v-chip
|
||||||
|
v-if="event.error"
|
||||||
|
color="error" size="x-small" variant="tonal"
|
||||||
|
prepend-icon="mdi-alert-octagon"
|
||||||
|
class="fc-dl-row__error"
|
||||||
|
:title="event.error"
|
||||||
|
>{{ truncateError(event.error) }}</v-chip>
|
||||||
|
<span v-else class="fc-dl-row__error-spacer" />
|
||||||
|
|
||||||
|
<div class="fc-dl-row__actions" @click.stop>
|
||||||
|
<v-btn
|
||||||
|
v-if="event.status === 'error' && event.source_id"
|
||||||
|
icon size="x-small" variant="text" color="warning"
|
||||||
|
:loading="retrying"
|
||||||
|
@click.stop="onRetry"
|
||||||
|
>
|
||||||
|
<v-icon size="small">mdi-refresh</v-icon>
|
||||||
|
<v-tooltip activator="parent" location="top">Retry source check</v-tooltip>
|
||||||
|
</v-btn>
|
||||||
|
<v-btn icon size="x-small" variant="text" @click.stop="$emit('open', event.id)">
|
||||||
|
<v-icon size="small">mdi-information-outline</v-icon>
|
||||||
|
<v-tooltip activator="parent" location="top">Details</v-tooltip>
|
||||||
|
</v-btn>
|
||||||
|
<v-btn
|
||||||
|
v-if="event.artist_slug"
|
||||||
|
icon size="x-small" variant="text"
|
||||||
|
:to="`/artist/${event.artist_slug}`"
|
||||||
|
@click.stop
|
||||||
|
>
|
||||||
|
<v-icon size="small">mdi-account-circle</v-icon>
|
||||||
|
<v-tooltip activator="parent" location="top">Open artist</v-tooltip>
|
||||||
|
</v-btn>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed } from 'vue'
|
import { toast } from '../../utils/toast.js'
|
||||||
|
import { computed, ref } from 'vue'
|
||||||
import { RouterLink } from 'vue-router'
|
import { RouterLink } from 'vue-router'
|
||||||
|
|
||||||
|
import PlatformChip from '../subscriptions/PlatformChip.vue'
|
||||||
|
import { useSourcesStore } from '../../stores/sources.js'
|
||||||
|
import { downloadStatusColor, downloadStatusIcon, downloadStatusLabel } from '../../utils/downloadStatus.js'
|
||||||
|
import { formatDateTime } from '../../utils/date.js'
|
||||||
|
|
||||||
const props = defineProps({ event: { type: Object, required: true } })
|
const props = defineProps({ event: { type: Object, required: true } })
|
||||||
defineEmits(['open'])
|
defineEmits(['open'])
|
||||||
|
|
||||||
const statusIcon = computed(() => ({
|
const sourcesStore = useSourcesStore()
|
||||||
ok: 'mdi-check-circle',
|
const retrying = ref(false)
|
||||||
error: 'mdi-alert-circle',
|
|
||||||
running: 'mdi-progress-clock',
|
|
||||||
pending: 'mdi-clock-outline',
|
|
||||||
skipped: 'mdi-minus-circle',
|
|
||||||
}[props.event.status] || 'mdi-help-circle'))
|
|
||||||
|
|
||||||
const statusColor = computed(() => ({
|
const statusColor = computed(() => downloadStatusColor(props.event.status))
|
||||||
ok: 'success',
|
const statusIcon = computed(() => downloadStatusIcon(props.event.status))
|
||||||
error: 'error',
|
const statusLabel = computed(() => downloadStatusLabel(props.event.status))
|
||||||
running: 'info',
|
|
||||||
pending: 'secondary',
|
|
||||||
skipped: 'warning',
|
|
||||||
}[props.event.status] || undefined))
|
|
||||||
|
|
||||||
function fmtTime(iso) {
|
function fmtTime(iso) {
|
||||||
if (!iso) return '—'
|
return iso ? formatDateTime(iso) : '—'
|
||||||
return iso.slice(0, 19).replace('T', ' ')
|
|
||||||
}
|
}
|
||||||
function fmtDuration(sec) {
|
function fmtDuration(sec) {
|
||||||
if (sec == null) return '—'
|
if (sec == null) return '—'
|
||||||
@@ -49,34 +114,107 @@ function fmtDuration(sec) {
|
|||||||
const m = Math.floor(sec / 60), s = Math.floor(sec % 60)
|
const m = Math.floor(sec / 60), s = Math.floor(sec % 60)
|
||||||
return `${m}m ${s}s`
|
return `${m}m ${s}s`
|
||||||
}
|
}
|
||||||
|
function truncateError(msg) {
|
||||||
|
const s = String(msg || '')
|
||||||
|
if (s.length <= 60) return s
|
||||||
|
return s.slice(0, 57) + '…'
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onRetry() {
|
||||||
|
if (!props.event.source_id) return
|
||||||
|
retrying.value = true
|
||||||
|
try {
|
||||||
|
await sourcesStore.checkNow(props.event.source_id)
|
||||||
|
toast({
|
||||||
|
text: `Source check re-queued`, type: 'success',
|
||||||
|
})
|
||||||
|
} catch (e) {
|
||||||
|
const isInFlight = !!e?.body?.download_event_id
|
||||||
|
toast({
|
||||||
|
text: isInFlight ? 'Already running' : `Retry failed: ${e?.detail || e?.message || e}`,
|
||||||
|
type: isInFlight ? 'info' : 'error',
|
||||||
|
})
|
||||||
|
} finally {
|
||||||
|
retrying.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
.fc-dl-row {
|
.fc-dl-row {
|
||||||
|
position: relative;
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: 24px 1fr 96px 160px 80px 80px 1fr;
|
grid-template-columns:
|
||||||
gap: 0.75rem;
|
/* bar */ 4px
|
||||||
|
/* status */ 120px
|
||||||
|
/* artist */ minmax(120px, 1.2fr)
|
||||||
|
/* plat */ 140px
|
||||||
|
/* time */ 140px
|
||||||
|
/* files */ 60px
|
||||||
|
/* dur */ 70px
|
||||||
|
/* error */ minmax(0, 1.5fr)
|
||||||
|
/* actions*/ 120px;
|
||||||
|
gap: 0.6rem;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
padding: 0.5rem 0.75rem;
|
padding: 0.55rem 0.75rem 0.55rem 0;
|
||||||
border-bottom: 1px solid rgb(var(--v-theme-on-surface-variant) / 0.18);
|
border-bottom: 1px solid rgb(var(--v-theme-on-surface-variant) / 0.15);
|
||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
|
transition: background 0.12s ease;
|
||||||
}
|
}
|
||||||
.fc-dl-row:hover { background: rgb(var(--v-theme-surface) / 0.5); }
|
.fc-dl-row:hover {
|
||||||
|
background: rgb(var(--v-theme-on-surface) / 0.04);
|
||||||
|
}
|
||||||
|
.fc-dl-row__bar {
|
||||||
|
width: 4px;
|
||||||
|
align-self: stretch;
|
||||||
|
border-radius: 0 2px 2px 0;
|
||||||
|
}
|
||||||
|
.fc-dl-row--ok .fc-dl-row__bar { background: rgb(var(--v-theme-success)); }
|
||||||
|
.fc-dl-row--error .fc-dl-row__bar { background: rgb(var(--v-theme-error)); }
|
||||||
|
.fc-dl-row--running .fc-dl-row__bar { background: rgb(var(--v-theme-info)); }
|
||||||
|
.fc-dl-row--skipped .fc-dl-row__bar { background: rgb(var(--v-theme-warning)); }
|
||||||
|
.fc-dl-row--pending .fc-dl-row__bar { background: rgb(var(--v-theme-on-surface-variant) / 0.4); }
|
||||||
|
|
||||||
|
.fc-dl-row__status { justify-self: start; }
|
||||||
.fc-dl-row__artist {
|
.fc-dl-row__artist {
|
||||||
color: rgb(var(--v-theme-on-surface));
|
color: rgb(var(--v-theme-on-surface));
|
||||||
text-decoration: none;
|
text-decoration: none;
|
||||||
|
font-weight: 500;
|
||||||
|
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||||
|
}
|
||||||
|
.fc-dl-row__artist--missing,
|
||||||
|
.fc-dl-row__platform-missing,
|
||||||
|
.fc-dl-row__no-files {
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
opacity: 0.5;
|
||||||
}
|
}
|
||||||
.fc-dl-row__artist:hover { color: rgb(var(--v-theme-accent)); }
|
.fc-dl-row__artist:hover { color: rgb(var(--v-theme-accent)); }
|
||||||
.fc-dl-row__time, .fc-dl-row__files, .fc-dl-row__duration {
|
.fc-dl-row__platform { justify-self: start; }
|
||||||
|
.fc-dl-row__time,
|
||||||
|
.fc-dl-row__duration {
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
font-size: 0.85rem;
|
font-size: 0.85rem;
|
||||||
font-variant-numeric: tabular-nums;
|
font-variant-numeric: tabular-nums;
|
||||||
}
|
|
||||||
.fc-dl-row__error {
|
|
||||||
color: rgb(var(--v-theme-error));
|
|
||||||
font-size: 0.85rem;
|
|
||||||
overflow: hidden;
|
|
||||||
text-overflow: ellipsis;
|
|
||||||
white-space: nowrap;
|
white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
.fc-dl-row__no-files {
|
||||||
|
text-align: center;
|
||||||
|
font-size: 1.1rem;
|
||||||
|
}
|
||||||
|
.fc-dl-row__error {
|
||||||
|
justify-self: start;
|
||||||
|
max-width: 100%;
|
||||||
|
}
|
||||||
|
.fc-dl-row__error :deep(.v-chip__content) {
|
||||||
|
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||||
|
}
|
||||||
|
.fc-dl-row__error-spacer { /* keeps the grid column reserved */ }
|
||||||
|
|
||||||
|
.fc-dl-row__actions {
|
||||||
|
display: flex; gap: 2px;
|
||||||
|
justify-self: end;
|
||||||
|
opacity: 0.5;
|
||||||
|
transition: opacity 0.12s ease;
|
||||||
|
}
|
||||||
|
.fc-dl-row:hover .fc-dl-row__actions { opacity: 1; }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -32,8 +32,9 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { onMounted, onUnmounted, ref, watch } from 'vue'
|
import { ref } from 'vue'
|
||||||
import { useGalleryStore } from '../../stores/gallery.js'
|
import { useGalleryStore } from '../../stores/gallery.js'
|
||||||
|
import { useInfiniteScroll } from '../../composables/useInfiniteScroll.js'
|
||||||
import GalleryItem from './GalleryItem.vue'
|
import GalleryItem from './GalleryItem.vue'
|
||||||
|
|
||||||
defineEmits(['open'])
|
defineEmits(['open'])
|
||||||
@@ -41,22 +42,9 @@ defineEmits(['open'])
|
|||||||
const store = useGalleryStore()
|
const store = useGalleryStore()
|
||||||
const sentinelEl = ref(null)
|
const sentinelEl = ref(null)
|
||||||
|
|
||||||
let observer = null
|
useInfiniteScroll(sentinelEl, () => {
|
||||||
|
if (store.hasMore && !store.loading) store.loadMore()
|
||||||
function attachObserver() {
|
})
|
||||||
if (observer) observer.disconnect()
|
|
||||||
if (!sentinelEl.value) return
|
|
||||||
observer = new IntersectionObserver(([entry]) => {
|
|
||||||
if (entry.isIntersecting && store.hasMore && !store.loading) {
|
|
||||||
store.loadMore()
|
|
||||||
}
|
|
||||||
}, { rootMargin: '600px' })
|
|
||||||
observer.observe(sentinelEl.value)
|
|
||||||
}
|
|
||||||
|
|
||||||
watch(sentinelEl, attachObserver)
|
|
||||||
onMounted(() => attachObserver())
|
|
||||||
onUnmounted(() => observer && observer.disconnect())
|
|
||||||
|
|
||||||
function dateHeaderId(group) { return `fc-month-${group.year}-${group.month}` }
|
function dateHeaderId(group) { return `fc-month-${group.year}-${group.month}` }
|
||||||
|
|
||||||
|
|||||||
@@ -19,7 +19,7 @@
|
|||||||
<span v-if="postDate(e)" class="fc-prov__date">{{ postDate(e) }}</span>
|
<span v-if="postDate(e)" class="fc-prov__date">{{ postDate(e) }}</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="fc-prov__post">
|
<div class="fc-prov__post">
|
||||||
{{ e.post.title || `Post ${e.post.external_post_id}` }}
|
{{ postTitle(e) }}
|
||||||
</div>
|
</div>
|
||||||
<div class="fc-prov__meta">
|
<div class="fc-prov__meta">
|
||||||
<RouterLink :to="`/artist/${e.artist.slug}`">
|
<RouterLink :to="`/artist/${e.artist.slug}`">
|
||||||
@@ -30,16 +30,16 @@
|
|||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="fc-prov__actions">
|
<div class="fc-prov__actions">
|
||||||
<a
|
|
||||||
v-if="e.post.url" :href="e.post.url"
|
|
||||||
target="_blank" rel="noopener noreferrer"
|
|
||||||
>↗ View original post</a>
|
|
||||||
<a href="#" @click.prevent="openPost(e.post.id)">
|
<a href="#" @click.prevent="openPost(e.post.id)">
|
||||||
View images from this post
|
View post
|
||||||
</a>
|
</a>
|
||||||
|
<a
|
||||||
|
v-if="e.post.description_html"
|
||||||
|
href="#" @click.prevent="toggleDesc(e.provenance_id)"
|
||||||
|
>{{ expanded[e.provenance_id] ? 'Hide description ▴' : 'Show description ▾' }}</a>
|
||||||
</div>
|
</div>
|
||||||
<div
|
<div
|
||||||
v-if="e.post.description_html"
|
v-if="e.post.description_html && expanded[e.provenance_id]"
|
||||||
class="fc-prov__desc" v-html="e.post.description_html"
|
class="fc-prov__desc" v-html="e.post.description_html"
|
||||||
/>
|
/>
|
||||||
</article>
|
</article>
|
||||||
@@ -74,16 +74,27 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed, watch } from 'vue'
|
import { computed, reactive, watch } from 'vue'
|
||||||
import { useRouter } from 'vue-router'
|
import { useRouter } from 'vue-router'
|
||||||
import { useModalStore } from '../../stores/modal.js'
|
import { useModalStore } from '../../stores/modal.js'
|
||||||
import { useProvenanceStore } from '../../stores/provenance.js'
|
import { useProvenanceStore } from '../../stores/provenance.js'
|
||||||
import { formatPostDate } from '../../utils/date.js'
|
import { formatPostDate } from '../../utils/date.js'
|
||||||
|
import { toPlainText } from '../../utils/htmlSanitize.js'
|
||||||
|
|
||||||
const modal = useModalStore()
|
const modal = useModalStore()
|
||||||
const prov = useProvenanceStore()
|
const prov = useProvenanceStore()
|
||||||
const router = useRouter()
|
const router = useRouter()
|
||||||
|
|
||||||
|
// Per-post description collapse state (keyed by provenance_id). Default
|
||||||
|
// collapsed so multiple posts don't each eat ~180px of the panel — the
|
||||||
|
// operator flagged the descriptions consuming a lot of real estate
|
||||||
|
// 2026-05-28. Reset when the viewed image changes.
|
||||||
|
const expanded = reactive({})
|
||||||
|
function toggleDesc(id) { expanded[id] = !expanded[id] }
|
||||||
|
watch(() => modal.currentImageId, () => {
|
||||||
|
for (const k of Object.keys(expanded)) delete expanded[k]
|
||||||
|
})
|
||||||
|
|
||||||
watch(
|
watch(
|
||||||
() => modal.currentImageId,
|
() => modal.currentImageId,
|
||||||
(id) => { if (id != null) prov.loadForImage(id) },
|
(id) => { if (id != null) prov.loadForImage(id) },
|
||||||
@@ -120,9 +131,16 @@ const show = computed(() => {
|
|||||||
const attachments = computed(() => state.value?.attachments || [])
|
const attachments = computed(() => state.value?.attachments || [])
|
||||||
|
|
||||||
function postDate(e) { return formatPostDate(e.post.date) }
|
function postDate(e) { return formatPostDate(e.post.date) }
|
||||||
|
function postTitle(e) {
|
||||||
|
// Titles can arrive as stored HTML (e.g. "<strong>…</strong>"); render
|
||||||
|
// as plain text (the CSS makes it bold).
|
||||||
|
return toPlainText(e.post.title) || `Post ${e.post.external_post_id}`
|
||||||
|
}
|
||||||
|
|
||||||
function openPost(postId) {
|
function openPost(postId) {
|
||||||
router.push({ path: '/gallery', query: { post_id: postId } })
|
// Land on the post in the posts feed (in context), not the gallery
|
||||||
|
// image grid. Operator-flagged 2026-05-28.
|
||||||
|
router.push({ path: '/posts', query: { post_id: postId } })
|
||||||
modal.close()
|
modal.close()
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
@@ -145,7 +163,7 @@ function openPost(postId) {
|
|||||||
text-transform: lowercase;
|
text-transform: lowercase;
|
||||||
}
|
}
|
||||||
.fc-prov__post {
|
.fc-prov__post {
|
||||||
font-weight: 600; margin: 4px 0;
|
font-weight: 700; margin: 4px 0;
|
||||||
color: rgb(var(--v-theme-on-surface));
|
color: rgb(var(--v-theme-on-surface));
|
||||||
}
|
}
|
||||||
.fc-prov__meta {
|
.fc-prov__meta {
|
||||||
|
|||||||
@@ -38,6 +38,7 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
import { computed, ref, watch } from 'vue'
|
import { computed, ref, watch } from 'vue'
|
||||||
import { useSuggestionsStore, CATEGORY_LABELS } from '../../stores/suggestions.js'
|
import { useSuggestionsStore, CATEGORY_LABELS } from '../../stores/suggestions.js'
|
||||||
import SuggestionsCategoryGroup from './SuggestionsCategoryGroup.vue'
|
import SuggestionsCategoryGroup from './SuggestionsCategoryGroup.vue'
|
||||||
@@ -57,7 +58,7 @@ watch(() => props.imageId, (id) => { if (id != null) store.load(id) }, { immedia
|
|||||||
|
|
||||||
async function onAccept(s) {
|
async function onAccept(s) {
|
||||||
try { await store.accept(s) }
|
try { await store.accept(s) }
|
||||||
catch (e) { window.__fcToast?.({ text: `Accept failed: ${e.message}`, type: 'error' }) }
|
catch (e) { toast({ text: `Accept failed: ${e.message}`, type: 'error' }) }
|
||||||
}
|
}
|
||||||
|
|
||||||
const aliasDialog = ref(false)
|
const aliasDialog = ref(false)
|
||||||
@@ -68,7 +69,7 @@ async function onAliasConfirm(canonicalTagId) {
|
|||||||
await store.aliasAccept(aliasTarget.value, canonicalTagId)
|
await store.aliasAccept(aliasTarget.value, canonicalTagId)
|
||||||
aliasDialog.value = false
|
aliasDialog.value = false
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
window.__fcToast?.({ text: `Alias failed: ${e.message}`, type: 'error' })
|
toast({ text: `Alias failed: ${e.message}`, type: 'error' })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -56,8 +56,8 @@
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="fc-post-card__text">
|
<div class="fc-post-card__text">
|
||||||
<h3 v-if="post.post_title" class="fc-post-card__title">
|
<h3 v-if="plainTitle" class="fc-post-card__title">
|
||||||
{{ post.post_title }}
|
{{ plainTitle }}
|
||||||
</h3>
|
</h3>
|
||||||
<h3 v-else class="fc-post-card__title fc-post-card__title--missing">
|
<h3 v-else class="fc-post-card__title fc-post-card__title--missing">
|
||||||
Post {{ post.external_post_id }}
|
Post {{ post.external_post_id }}
|
||||||
@@ -80,8 +80,8 @@
|
|||||||
<!-- Expanded body: title, full mosaic, full sanitized HTML description,
|
<!-- Expanded body: title, full mosaic, full sanitized HTML description,
|
||||||
attachments. Lazy-loaded detail via getPostFull. -->
|
attachments. Lazy-loaded detail via getPostFull. -->
|
||||||
<div v-else class="fc-post-card__expanded">
|
<div v-else class="fc-post-card__expanded">
|
||||||
<h2 v-if="post.post_title" class="fc-post-card__title-full">
|
<h2 v-if="plainTitle" class="fc-post-card__title-full">
|
||||||
{{ post.post_title }}
|
{{ plainTitle }}
|
||||||
</h2>
|
</h2>
|
||||||
<h2 v-else class="fc-post-card__title-full fc-post-card__title--missing">
|
<h2 v-else class="fc-post-card__title-full fc-post-card__title--missing">
|
||||||
Post {{ post.external_post_id }}
|
Post {{ post.external_post_id }}
|
||||||
@@ -125,7 +125,7 @@ import { computed, ref } from 'vue'
|
|||||||
import { RouterLink } from 'vue-router'
|
import { RouterLink } from 'vue-router'
|
||||||
|
|
||||||
import { usePostsStore } from '../../stores/posts.js'
|
import { usePostsStore } from '../../stores/posts.js'
|
||||||
import { sanitizeHtml } from '../../utils/htmlSanitize.js'
|
import { sanitizeHtml, toPlainText } from '../../utils/htmlSanitize.js'
|
||||||
import PostEmptyThumbs from './PostEmptyThumbs.vue'
|
import PostEmptyThumbs from './PostEmptyThumbs.vue'
|
||||||
import PostImageGrid from './PostImageGrid.vue'
|
import PostImageGrid from './PostImageGrid.vue'
|
||||||
|
|
||||||
@@ -149,6 +149,10 @@ const merged = computed(() => detail.value || props.post)
|
|||||||
const images = computed(() => merged.value.thumbnails || [])
|
const images = computed(() => merged.value.thumbnails || [])
|
||||||
const attachments = computed(() => merged.value.attachments || [])
|
const attachments = computed(() => merged.value.attachments || [])
|
||||||
|
|
||||||
|
// Titles can arrive as stored HTML (e.g. "<strong>…</strong>"); render as
|
||||||
|
// plain text — the CSS makes the title bold.
|
||||||
|
const plainTitle = computed(() => toPlainText(props.post.post_title))
|
||||||
|
|
||||||
// Compact-view hero+rail derived from the feed-shape (capped 6).
|
// Compact-view hero+rail derived from the feed-shape (capped 6).
|
||||||
const hero = computed(() => props.post.thumbnails?.[0])
|
const hero = computed(() => props.post.thumbnails?.[0])
|
||||||
const rail = computed(() => (props.post.thumbnails || []).slice(1, 4))
|
const rail = computed(() => (props.post.thumbnails || []).slice(1, 4))
|
||||||
@@ -312,7 +316,7 @@ function formatBytes (n) {
|
|||||||
|
|
||||||
.fc-post-card__title {
|
.fc-post-card__title {
|
||||||
font-family: 'Fraunces', Georgia, serif;
|
font-family: 'Fraunces', Georgia, serif;
|
||||||
font-size: 18px; font-weight: 500;
|
font-size: 18px; font-weight: 700;
|
||||||
margin: 0 0 8px 0;
|
margin: 0 0 8px 0;
|
||||||
color: rgb(var(--v-theme-on-surface));
|
color: rgb(var(--v-theme-on-surface));
|
||||||
display: -webkit-box;
|
display: -webkit-box;
|
||||||
@@ -369,7 +373,7 @@ function formatBytes (n) {
|
|||||||
.fc-post-card__title-full {
|
.fc-post-card__title-full {
|
||||||
font-family: 'Fraunces', Georgia, serif;
|
font-family: 'Fraunces', Georgia, serif;
|
||||||
font-size: 22px;
|
font-size: 22px;
|
||||||
font-weight: 500;
|
font-weight: 700;
|
||||||
margin: 0;
|
margin: 0;
|
||||||
color: rgb(var(--v-theme-on-surface));
|
color: rgb(var(--v-theme-on-surface));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -59,6 +59,8 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { formatRelative as fmtRelative } from '../../utils/date.js'
|
||||||
|
|
||||||
defineProps({ runs: { type: Array, default: () => [] } })
|
defineProps({ runs: { type: Array, default: () => [] } })
|
||||||
defineEmits(['restore', 'delete', 'tag'])
|
defineEmits(['restore', 'delete', 'tag'])
|
||||||
|
|
||||||
@@ -92,13 +94,7 @@ function formatBytes(b) {
|
|||||||
return `${v.toFixed(i === 0 ? 0 : 1)} ${units[i]}`
|
return `${v.toFixed(i === 0 ? 0 : 1)} ${units[i]}`
|
||||||
}
|
}
|
||||||
function formatRelative(iso) {
|
function formatRelative(iso) {
|
||||||
if (!iso) return '—'
|
return fmtRelative(iso, { nullText: '—' })
|
||||||
const then = new Date(iso).getTime()
|
|
||||||
const diff = Math.max(0, (Date.now() - then) / 1000)
|
|
||||||
if (diff < 60) return `${Math.floor(diff)}s ago`
|
|
||||||
if (diff < 3600) return `${Math.floor(diff / 60)}m ago`
|
|
||||||
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`
|
|
||||||
return `${Math.floor(diff / 86400)}d ago`
|
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|||||||
@@ -108,6 +108,7 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
import { computed, onMounted, ref } from 'vue'
|
import { computed, onMounted, ref } from 'vue'
|
||||||
import { useApi } from '../../composables/useApi.js'
|
import { useApi } from '../../composables/useApi.js'
|
||||||
import { copyText } from '../../utils/clipboard.js'
|
import { copyText } from '../../utils/clipboard.js'
|
||||||
@@ -147,7 +148,7 @@ async function loadKey() {
|
|||||||
apiKey.value = key
|
apiKey.value = key
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
apiKey.value = ''
|
apiKey.value = ''
|
||||||
window.__fcToast?.({
|
toast({
|
||||||
text: `Failed to load extension API key: ${e.message}`,
|
text: `Failed to load extension API key: ${e.message}`,
|
||||||
type: 'error',
|
type: 'error',
|
||||||
})
|
})
|
||||||
@@ -160,9 +161,9 @@ async function rotateKey() {
|
|||||||
const { key } = await api.post('/api/settings/extension_api_key/rotate')
|
const { key } = await api.post('/api/settings/extension_api_key/rotate')
|
||||||
apiKey.value = key
|
apiKey.value = key
|
||||||
keyShown.value = true
|
keyShown.value = true
|
||||||
window.__fcToast?.({ text: 'Extension API key rotated.', type: 'success' })
|
toast({ text: 'Extension API key rotated.', type: 'success' })
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
window.__fcToast?.({
|
toast({
|
||||||
text: `Rotate failed: ${e.message}`,
|
text: `Rotate failed: ${e.message}`,
|
||||||
type: 'error',
|
type: 'error',
|
||||||
})
|
})
|
||||||
@@ -174,9 +175,9 @@ async function rotateKey() {
|
|||||||
async function copy(text, label) {
|
async function copy(text, label) {
|
||||||
try {
|
try {
|
||||||
await copyText(text)
|
await copyText(text)
|
||||||
window.__fcToast?.({ text: `${label} copied.`, type: 'success' })
|
toast({ text: `${label} copied.`, type: 'success' })
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
window.__fcToast?.({ text: `Copy failed: ${e.message}`, type: 'error' })
|
toast({ text: `Copy failed: ${e.message}`, type: 'error' })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -11,20 +11,23 @@
|
|||||||
<v-icon start>mdi-vector-triangle</v-icon> Recompute centroids
|
<v-icon start>mdi-vector-triangle</v-icon> Recompute centroids
|
||||||
</v-btn>
|
</v-btn>
|
||||||
<span v-if="done" class="ml-3 text-caption">Enqueued.</span>
|
<span v-if="done" class="ml-3 text-caption">Enqueued.</span>
|
||||||
|
<QueueStatusBar queue="ml" queue-label="ML" />
|
||||||
</v-card-text>
|
</v-card-text>
|
||||||
</v-card>
|
</v-card>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
import { ref } from 'vue'
|
import { ref } from 'vue'
|
||||||
import { useMLStore } from '../../stores/ml.js'
|
import { useMLStore } from '../../stores/ml.js'
|
||||||
|
import QueueStatusBar from './QueueStatusBar.vue'
|
||||||
const store = useMLStore()
|
const store = useMLStore()
|
||||||
const busy = ref(false)
|
const busy = ref(false)
|
||||||
const done = ref(false)
|
const done = ref(false)
|
||||||
async function run() {
|
async function run() {
|
||||||
busy.value = true
|
busy.value = true
|
||||||
try { await store.triggerRecomputeCentroids(); done.value = true }
|
try { await store.triggerRecomputeCentroids(); done.value = true }
|
||||||
catch (e) { window.__fcToast?.({ text: e.message, type: 'error' }) }
|
catch (e) { toast({ text: e.message, type: 'error' }) }
|
||||||
finally { busy.value = false }
|
finally { busy.value = false }
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -51,6 +51,19 @@
|
|||||||
title="Click for full error"
|
title="Click for full error"
|
||||||
>{{ shorten(item.error, 60) }}</button>
|
>{{ shorten(item.error, 60) }}</button>
|
||||||
</template>
|
</template>
|
||||||
|
<template #item.actions="{ item }">
|
||||||
|
<v-btn
|
||||||
|
v-if="item.status === 'failed'"
|
||||||
|
icon size="x-small" variant="text"
|
||||||
|
:loading="refetching === item.id"
|
||||||
|
@click="onRefetch(item)"
|
||||||
|
>
|
||||||
|
<v-icon size="small">mdi-cloud-refresh</v-icon>
|
||||||
|
<v-tooltip activator="parent" location="top">
|
||||||
|
Re-fetch original (re-download from source)
|
||||||
|
</v-tooltip>
|
||||||
|
</v-btn>
|
||||||
|
</template>
|
||||||
</v-data-table-virtual>
|
</v-data-table-virtual>
|
||||||
<div v-if="store.hasMore" class="d-flex justify-center py-3">
|
<div v-if="store.hasMore" class="d-flex justify-center py-3">
|
||||||
<v-btn variant="text" size="small" @click="onLoadMore">Load more</v-btn>
|
<v-btn variant="text" size="small" @click="onLoadMore">Load more</v-btn>
|
||||||
@@ -112,6 +125,7 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
import { computed, ref } from 'vue'
|
import { computed, ref } from 'vue'
|
||||||
import { useImportStore } from '../../stores/import.js'
|
import { useImportStore } from '../../stores/import.js'
|
||||||
import ErrorDetailModal from '../common/ErrorDetailModal.vue'
|
import ErrorDetailModal from '../common/ErrorDetailModal.vue'
|
||||||
@@ -149,9 +163,29 @@ const headers = [
|
|||||||
{ title: 'Source', key: 'source_path', sortable: false },
|
{ title: 'Source', key: 'source_path', sortable: false },
|
||||||
{ title: 'Size', key: 'size_bytes', sortable: false, width: 90 },
|
{ title: 'Size', key: 'size_bytes', sortable: false, width: 90 },
|
||||||
{ title: 'Created', key: 'created_at', sortable: false, width: 150 },
|
{ title: 'Created', key: 'created_at', sortable: false, width: 150 },
|
||||||
{ title: 'Note', key: 'error', sortable: false }
|
{ title: 'Note', key: 'error', sortable: false },
|
||||||
|
{ title: '', key: 'actions', sortable: false, width: 56 }
|
||||||
]
|
]
|
||||||
|
|
||||||
|
const refetching = ref(null)
|
||||||
|
const _REFETCH_MSG = {
|
||||||
|
refetch_queued: { text: 'Re-fetch queued — re-downloading from source', type: 'success' },
|
||||||
|
no_source: { text: 'No re-fetchable source (filesystem import — replace the file manually)', type: 'info' },
|
||||||
|
already_refetched: { text: 'Already re-fetched once', type: 'info' },
|
||||||
|
}
|
||||||
|
async function onRefetch(item) {
|
||||||
|
refetching.value = item.id
|
||||||
|
try {
|
||||||
|
const res = await store.refetchTask(item.id)
|
||||||
|
const msg = _REFETCH_MSG[res.status] || { text: `Re-fetch: ${res.status}`, type: 'info' }
|
||||||
|
toast(msg)
|
||||||
|
} catch (e) {
|
||||||
|
toast({ text: `Re-fetch failed: ${e.message}`, type: 'error' })
|
||||||
|
} finally {
|
||||||
|
refetching.value = null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const hasFailed = computed(() => store.tasks.some(t => t.status === 'failed'))
|
const hasFailed = computed(() => store.tasks.some(t => t.status === 'failed'))
|
||||||
const hasStuck = computed(() => store.tasks.some(
|
const hasStuck = computed(() => store.tasks.some(
|
||||||
t => t.status === 'pending' || t.status === 'queued' || t.status === 'processing'
|
t => t.status === 'pending' || t.status === 'queued' || t.status === 'processing'
|
||||||
|
|||||||
@@ -1,245 +0,0 @@
|
|||||||
<template>
|
|
||||||
<v-card>
|
|
||||||
<v-card-title>Legacy migration</v-card-title>
|
|
||||||
<v-card-text>
|
|
||||||
<p class="text-body-2 mb-3">
|
|
||||||
Migrate existing ImageRepo and GallerySubscriber data into FabledCurator.
|
|
||||||
Each app produces a JSON export file via a small script in its own repo
|
|
||||||
(<code>scripts/export_for_fabledcurator.py</code>); upload both files
|
|
||||||
here and run the steps in order.
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<ol class="text-body-2 mb-3 fc-migrate__hints">
|
|
||||||
<li>Backup FC (creates a pre-migration snapshot).</li>
|
|
||||||
<li>Upload + ingest GS export.</li>
|
|
||||||
<li>Upload + ingest IR export.</li>
|
|
||||||
<li>Switch to <strong>Import tab</strong> → trigger the existing FC filesystem scan over the bind-mounted IR images dir.</li>
|
|
||||||
<li>Return here → apply IR tag associations (joins by sha256).</li>
|
|
||||||
<li>Queue ML re-processing.</li>
|
|
||||||
<li>Verify.</li>
|
|
||||||
</ol>
|
|
||||||
|
|
||||||
<div class="fc-migrate__uploads">
|
|
||||||
<v-file-input
|
|
||||||
v-model="gsFile"
|
|
||||||
label="GallerySubscriber export (gs-export.json)"
|
|
||||||
accept="application/json,.json"
|
|
||||||
density="compact"
|
|
||||||
show-size
|
|
||||||
prepend-icon="mdi-upload"
|
|
||||||
/>
|
|
||||||
<v-file-input
|
|
||||||
v-model="irFile"
|
|
||||||
label="ImageRepo export (ir-export.json)"
|
|
||||||
accept="application/json,.json"
|
|
||||||
density="compact"
|
|
||||||
show-size
|
|
||||||
prepend-icon="mdi-upload"
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="fc-migrate__steps mt-3">
|
|
||||||
<v-btn
|
|
||||||
color="accent" variant="tonal" prepend-icon="mdi-content-save"
|
|
||||||
:disabled="store.isRunning"
|
|
||||||
@click="onBackup"
|
|
||||||
>1. Backup FC</v-btn>
|
|
||||||
<v-btn
|
|
||||||
color="primary" variant="tonal" prepend-icon="mdi-database-arrow-right"
|
|
||||||
:disabled="store.isRunning || !gsFile"
|
|
||||||
@click="onIngestGs"
|
|
||||||
>2. Ingest GS</v-btn>
|
|
||||||
<v-btn
|
|
||||||
color="primary" variant="tonal" prepend-icon="mdi-database-arrow-right"
|
|
||||||
:disabled="store.isRunning || !irFile"
|
|
||||||
@click="onIngestIr"
|
|
||||||
>3. Ingest IR</v-btn>
|
|
||||||
<v-btn
|
|
||||||
color="primary" variant="tonal" prepend-icon="mdi-tag-multiple"
|
|
||||||
:disabled="store.isRunning"
|
|
||||||
@click="onTagApply"
|
|
||||||
>5. Apply IR tags</v-btn>
|
|
||||||
<v-btn
|
|
||||||
variant="outlined" prepend-icon="mdi-brain"
|
|
||||||
:disabled="store.isRunning"
|
|
||||||
@click="onMlQueue"
|
|
||||||
>6. Queue ML</v-btn>
|
|
||||||
<v-btn
|
|
||||||
variant="outlined" prepend-icon="mdi-check-circle"
|
|
||||||
:disabled="store.isRunning"
|
|
||||||
@click="onVerify"
|
|
||||||
>7. Verify</v-btn>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<v-btn
|
|
||||||
color="error" variant="outlined" prepend-icon="mdi-restore" class="mt-3"
|
|
||||||
:disabled="store.isRunning"
|
|
||||||
@click="onRollback"
|
|
||||||
>Rollback to pre-migration backup</v-btn>
|
|
||||||
|
|
||||||
<v-alert v-if="store.error" type="error" variant="tonal" class="mt-3" closable>
|
|
||||||
{{ String(store.error) }}
|
|
||||||
</v-alert>
|
|
||||||
|
|
||||||
<v-card v-if="store.activeRun" variant="outlined" class="mt-4">
|
|
||||||
<v-card-text>
|
|
||||||
<div class="text-subtitle-2">
|
|
||||||
#{{ store.activeRun.id }} — {{ store.activeRun.kind }}
|
|
||||||
({{ store.activeRun.status }})
|
|
||||||
</div>
|
|
||||||
<v-progress-linear
|
|
||||||
v-if="store.isRunning" indeterminate color="accent" class="my-2"
|
|
||||||
/>
|
|
||||||
<div class="text-caption">
|
|
||||||
Rows: {{ store.activeRun.counts.rows_processed || 0 }} processed,
|
|
||||||
{{ store.activeRun.counts.rows_inserted || 0 }} inserted,
|
|
||||||
{{ store.activeRun.counts.rows_skipped || 0 }} skipped.
|
|
||||||
Conflicts: {{ store.activeRun.counts.conflicts || 0 }}.
|
|
||||||
</div>
|
|
||||||
<div v-if="store.activeRun.error" class="text-error mt-1">
|
|
||||||
{{ store.activeRun.error }}
|
|
||||||
</div>
|
|
||||||
</v-card-text>
|
|
||||||
</v-card>
|
|
||||||
|
|
||||||
<div v-if="store.recentRuns.length" class="mt-4">
|
|
||||||
<div class="text-subtitle-2 mb-2">Recent runs</div>
|
|
||||||
<ul class="fc-migrate__history">
|
|
||||||
<li v-for="r in store.recentRuns" :key="r.id">
|
|
||||||
#{{ r.id }} {{ r.kind }}
|
|
||||||
<v-chip
|
|
||||||
size="x-small"
|
|
||||||
:color="r.status === 'ok' ? 'success' : (r.status === 'error' ? 'error' : undefined)"
|
|
||||||
>{{ r.status }}</v-chip>
|
|
||||||
<span class="text-caption fc-migrate__when">{{ r.started_at }}</span>
|
|
||||||
</li>
|
|
||||||
</ul>
|
|
||||||
</div>
|
|
||||||
</v-card-text>
|
|
||||||
|
|
||||||
<v-dialog v-model="confirmOpen" max-width="520">
|
|
||||||
<v-card>
|
|
||||||
<v-card-title>Confirm {{ pendingLabel }}</v-card-title>
|
|
||||||
<v-card-text>
|
|
||||||
This will modify FC's database. Make sure you've taken a backup first.
|
|
||||||
Continue?
|
|
||||||
</v-card-text>
|
|
||||||
<v-card-actions>
|
|
||||||
<v-spacer />
|
|
||||||
<v-btn variant="text" @click="confirmOpen = false">Cancel</v-btn>
|
|
||||||
<v-btn color="primary" variant="tonal" @click="onConfirm">Continue</v-btn>
|
|
||||||
</v-card-actions>
|
|
||||||
</v-card>
|
|
||||||
</v-dialog>
|
|
||||||
</v-card>
|
|
||||||
</template>
|
|
||||||
|
|
||||||
<script setup>
|
|
||||||
import { onMounted, ref } from 'vue'
|
|
||||||
import { useMigrationStore } from '../../stores/migration.js'
|
|
||||||
|
|
||||||
const store = useMigrationStore()
|
|
||||||
|
|
||||||
const gsFile = ref(null)
|
|
||||||
const irFile = ref(null)
|
|
||||||
|
|
||||||
const confirmOpen = ref(false)
|
|
||||||
const pendingLabel = ref('')
|
|
||||||
const pendingAction = ref(null)
|
|
||||||
|
|
||||||
onMounted(async () => {
|
|
||||||
await store.loadRecent()
|
|
||||||
})
|
|
||||||
|
|
||||||
function _pick(model) {
|
|
||||||
if (!model) return null
|
|
||||||
// v-file-input v-model can be a single File or a [File] depending on Vuetify version.
|
|
||||||
return Array.isArray(model) ? model[0] : model
|
|
||||||
}
|
|
||||||
|
|
||||||
function requestConfirm(label, action) {
|
|
||||||
pendingLabel.value = label
|
|
||||||
pendingAction.value = action
|
|
||||||
confirmOpen.value = true
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onConfirm() {
|
|
||||||
confirmOpen.value = false
|
|
||||||
const action = pendingAction.value
|
|
||||||
pendingAction.value = null
|
|
||||||
if (action) await action()
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onBackup() {
|
|
||||||
await store.trigger('backup', { tag: 'pre_migration' })
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onIngestGs() {
|
|
||||||
const file = _pick(gsFile.value)
|
|
||||||
if (!file) return
|
|
||||||
requestConfirm('Ingest GS', async () => {
|
|
||||||
await store.trigger('gs_ingest', { dry_run: false }, file)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onIngestIr() {
|
|
||||||
const file = _pick(irFile.value)
|
|
||||||
if (!file) return
|
|
||||||
requestConfirm('Ingest IR', async () => {
|
|
||||||
await store.trigger('ir_ingest', { dry_run: false }, file)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onTagApply() {
|
|
||||||
requestConfirm('Apply IR tag associations', async () => {
|
|
||||||
await store.trigger('tag_apply', { dry_run: false })
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onMlQueue() {
|
|
||||||
await store.trigger('ml_queue', {})
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onVerify() {
|
|
||||||
await store.trigger('verify', {})
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onRollback() {
|
|
||||||
requestConfirm('Rollback to pre-migration backup', async () => {
|
|
||||||
await store.trigger('rollback', {})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
</script>
|
|
||||||
|
|
||||||
<style scoped>
|
|
||||||
.fc-migrate__hints {
|
|
||||||
padding-left: 1.2rem;
|
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
|
||||||
}
|
|
||||||
.fc-migrate__uploads {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: 1fr 1fr;
|
|
||||||
gap: 12px;
|
|
||||||
}
|
|
||||||
.fc-migrate__steps {
|
|
||||||
display: flex;
|
|
||||||
flex-wrap: wrap;
|
|
||||||
gap: 8px;
|
|
||||||
}
|
|
||||||
.fc-migrate__history {
|
|
||||||
list-style: none;
|
|
||||||
padding: 0;
|
|
||||||
margin: 0;
|
|
||||||
}
|
|
||||||
.fc-migrate__history li {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 8px;
|
|
||||||
padding: 4px 0;
|
|
||||||
border-bottom: 1px solid rgb(var(--v-theme-on-surface-variant), 0.15);
|
|
||||||
}
|
|
||||||
.fc-migrate__when {
|
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
|
||||||
margin-left: auto;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
@@ -10,20 +10,23 @@
|
|||||||
<v-icon start>mdi-refresh</v-icon> Run backfill now
|
<v-icon start>mdi-refresh</v-icon> Run backfill now
|
||||||
</v-btn>
|
</v-btn>
|
||||||
<span v-if="done" class="ml-3 text-caption">Enqueued.</span>
|
<span v-if="done" class="ml-3 text-caption">Enqueued.</span>
|
||||||
|
<QueueStatusBar queue="ml" queue-label="ML" />
|
||||||
</v-card-text>
|
</v-card-text>
|
||||||
</v-card>
|
</v-card>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
import { ref } from 'vue'
|
import { ref } from 'vue'
|
||||||
import { useMLStore } from '../../stores/ml.js'
|
import { useMLStore } from '../../stores/ml.js'
|
||||||
|
import QueueStatusBar from './QueueStatusBar.vue'
|
||||||
const store = useMLStore()
|
const store = useMLStore()
|
||||||
const busy = ref(false)
|
const busy = ref(false)
|
||||||
const done = ref(false)
|
const done = ref(false)
|
||||||
async function run() {
|
async function run() {
|
||||||
busy.value = true
|
busy.value = true
|
||||||
try { await store.triggerBackfill(); done.value = true }
|
try { await store.triggerBackfill(); done.value = true }
|
||||||
catch (e) { window.__fcToast?.({ text: e.message, type: 'error' }) }
|
catch (e) { toast({ text: e.message, type: 'error' }) }
|
||||||
finally { busy.value = false }
|
finally { busy.value = false }
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -17,6 +17,7 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
import { reactive, watch } from 'vue'
|
import { reactive, watch } from 'vue'
|
||||||
import { useMLStore } from '../../stores/ml.js'
|
import { useMLStore } from '../../stores/ml.js'
|
||||||
|
|
||||||
@@ -35,6 +36,6 @@ async function save() {
|
|||||||
const patch = {}
|
const patch = {}
|
||||||
for (const f of fields) patch[f.key] = local[f.key]
|
for (const f of fields) patch[f.key] = local[f.key]
|
||||||
try { await store.patchSettings(patch) }
|
try { await store.patchSettings(patch) }
|
||||||
catch (e) { window.__fcToast?.({ text: e.message, type: 'error' }) }
|
catch (e) { toast({ text: e.message, type: 'error' }) }
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -17,12 +17,14 @@
|
|||||||
<BackupCard class="mt-6" />
|
<BackupCard class="mt-6" />
|
||||||
<!-- TagMaintenanceCard moved to Cleanup tab (v26.05.25.7) — it
|
<!-- TagMaintenanceCard moved to Cleanup tab (v26.05.25.7) — it
|
||||||
operates on the existing library which fits the Cleanup-tab
|
operates on the existing library which fits the Cleanup-tab
|
||||||
theme, and clusters with the other audit cards. -->
|
theme, and clusters with the other audit cards. LegacyMigrationCard
|
||||||
<LegacyMigrationCard class="mt-6" />
|
removed once the one-and-done GS/IR migration cutover completed. -->
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { onMounted, onUnmounted } from 'vue'
|
||||||
|
|
||||||
import MLBackfillCard from './MLBackfillCard.vue'
|
import MLBackfillCard from './MLBackfillCard.vue'
|
||||||
import CentroidRecomputeCard from './CentroidRecomputeCard.vue'
|
import CentroidRecomputeCard from './CentroidRecomputeCard.vue'
|
||||||
import ThumbnailBackfillCard from './ThumbnailBackfillCard.vue'
|
import ThumbnailBackfillCard from './ThumbnailBackfillCard.vue'
|
||||||
@@ -30,7 +32,22 @@ import MLThresholdSliders from './MLThresholdSliders.vue'
|
|||||||
import AllowlistTable from './AllowlistTable.vue'
|
import AllowlistTable from './AllowlistTable.vue'
|
||||||
import AliasTable from './AliasTable.vue'
|
import AliasTable from './AliasTable.vue'
|
||||||
import BackupCard from './BackupCard.vue'
|
import BackupCard from './BackupCard.vue'
|
||||||
import LegacyMigrationCard from './LegacyMigrationCard.vue'
|
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||||
|
|
||||||
|
// Poll queue depths so each card's QueueStatusBar shows live pending
|
||||||
|
// counts — the operator can see a backfill is already queued/running
|
||||||
|
// before re-triggering it.
|
||||||
|
const activity = useSystemActivityStore()
|
||||||
|
let qTimer = null
|
||||||
|
onMounted(() => {
|
||||||
|
activity.loadQueues()
|
||||||
|
qTimer = setInterval(() => {
|
||||||
|
if (!document.hidden) activity.loadQueues()
|
||||||
|
}, 4000)
|
||||||
|
})
|
||||||
|
onUnmounted(() => {
|
||||||
|
if (qTimer) { clearInterval(qTimer); qTimer = null }
|
||||||
|
})
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
|||||||
@@ -0,0 +1,69 @@
|
|||||||
|
<template>
|
||||||
|
<div class="fc-qbar" :class="`fc-qbar--${state}`" :title="hint">
|
||||||
|
<v-icon size="14" class="fc-qbar__icon">{{ icon }}</v-icon>
|
||||||
|
<span>{{ label }}</span>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed } from 'vue'
|
||||||
|
|
||||||
|
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||||
|
|
||||||
|
const props = defineProps({
|
||||||
|
// Celery queue these buttons feed (see celery_app.task_routes).
|
||||||
|
queue: { type: String, required: true },
|
||||||
|
queueLabel: { type: String, default: '' },
|
||||||
|
})
|
||||||
|
|
||||||
|
const store = useSystemActivityStore()
|
||||||
|
|
||||||
|
// Redis LLEN per queue from /api/system/activity/queues; null when the
|
||||||
|
// broker read failed or hasn't loaded yet.
|
||||||
|
const depth = computed(() => {
|
||||||
|
const q = store.queues?.queues
|
||||||
|
return q ? (q[props.queue] ?? null) : null
|
||||||
|
})
|
||||||
|
const state = computed(() => {
|
||||||
|
if (depth.value == null) return 'unknown'
|
||||||
|
return depth.value > 0 ? 'busy' : 'idle'
|
||||||
|
})
|
||||||
|
const icon = computed(() => ({
|
||||||
|
busy: 'mdi-progress-clock',
|
||||||
|
idle: 'mdi-check-circle-outline',
|
||||||
|
unknown: 'mdi-help-circle-outline',
|
||||||
|
}[state.value]))
|
||||||
|
const qname = computed(() => props.queueLabel || props.queue)
|
||||||
|
const label = computed(() => {
|
||||||
|
if (depth.value == null) return `${qname.value} queue · status unavailable`
|
||||||
|
if (depth.value === 0) return `${qname.value} queue idle — nothing pending`
|
||||||
|
return `${depth.value} pending in the ${qname.value} queue`
|
||||||
|
})
|
||||||
|
const hint = computed(() =>
|
||||||
|
state.value === 'busy'
|
||||||
|
? `The ${qname.value} queue already has ${depth.value} task(s) waiting — running again now just adds to that backlog.`
|
||||||
|
: '',
|
||||||
|
)
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-qbar {
|
||||||
|
display: flex; align-items: center; gap: 6px;
|
||||||
|
margin-top: 12px;
|
||||||
|
padding: 5px 10px;
|
||||||
|
border-radius: 6px;
|
||||||
|
font-size: 0.78rem;
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
}
|
||||||
|
.fc-qbar__icon { flex: 0 0 auto; }
|
||||||
|
.fc-qbar--idle,
|
||||||
|
.fc-qbar--unknown {
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
background: rgb(var(--v-theme-on-surface) / 0.04);
|
||||||
|
}
|
||||||
|
.fc-qbar--busy {
|
||||||
|
color: rgb(var(--v-theme-warning));
|
||||||
|
background: rgb(var(--v-theme-warning) / 0.12);
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -154,6 +154,7 @@
|
|||||||
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||||
|
|
||||||
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||||
|
import { formatRelative as fmtRelative } from '../../utils/date.js'
|
||||||
import ErrorDetailModal from '../common/ErrorDetailModal.vue'
|
import ErrorDetailModal from '../common/ErrorDetailModal.vue'
|
||||||
import QueuesTable from './QueuesTable.vue'
|
import QueuesTable from './QueuesTable.vue'
|
||||||
|
|
||||||
@@ -277,14 +278,7 @@ function formatDuration(ms) {
|
|||||||
return `${(ms / 60_000).toFixed(1)} min`
|
return `${(ms / 60_000).toFixed(1)} min`
|
||||||
}
|
}
|
||||||
function formatRelative(iso) {
|
function formatRelative(iso) {
|
||||||
if (!iso) return '—'
|
return fmtRelative(iso, { nullText: '—' })
|
||||||
const then = new Date(iso).getTime()
|
|
||||||
const now = Date.now()
|
|
||||||
const diff = Math.max(0, (now - then) / 1000)
|
|
||||||
if (diff < 60) return `${Math.floor(diff)}s ago`
|
|
||||||
if (diff < 3600) return `${Math.floor(diff / 60)}m ago`
|
|
||||||
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`
|
|
||||||
return `${Math.floor(diff / 86400)}d ago`
|
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|||||||
@@ -44,6 +44,51 @@
|
|||||||
@click="onCommit"
|
@click="onCommit"
|
||||||
>Delete {{ preview.count }} unused tag(s)</v-btn>
|
>Delete {{ preview.count }} unused tag(s)</v-btn>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<v-divider class="my-5" />
|
||||||
|
|
||||||
|
<p class="fc-muted text-body-2 mb-4">
|
||||||
|
Purge legacy IR-migration tags FC no longer uses: retired/system
|
||||||
|
kinds (<code>archive</code>, <code>post</code>,
|
||||||
|
<code>artist</code> — e.g.
|
||||||
|
<code>BlenderKnight:Hannah_BJ_Loops</code>) plus
|
||||||
|
<code>source:*</code> tags (ImageRepo's old <code>source</code>
|
||||||
|
kind, which migrated to <code>general</code>). Provenance and
|
||||||
|
artists are their own systems now, so these are pure noise.
|
||||||
|
Removes them from every image.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<v-btn
|
||||||
|
color="accent" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-magnify"
|
||||||
|
:loading="loadingKindPreview"
|
||||||
|
class="mb-3"
|
||||||
|
@click="onKindPreview"
|
||||||
|
>Preview legacy tags</v-btn>
|
||||||
|
|
||||||
|
<div v-if="kindPreview">
|
||||||
|
<p class="text-body-2 mb-2">
|
||||||
|
<strong>{{ kindPreview.count }}</strong> legacy tag(s).
|
||||||
|
<span v-for="(n, k) in kindPreview.by_kind" :key="k" class="fc-muted">
|
||||||
|
{{ k }}: {{ n }}
|
||||||
|
</span>
|
||||||
|
<span v-for="(n, p) in kindPreview.by_prefix" :key="p" class="fc-muted">
|
||||||
|
{{ p }}: {{ n }}
|
||||||
|
</span>
|
||||||
|
</p>
|
||||||
|
<div v-if="kindPreview.sample_names?.length" class="fc-name-grid mb-3">
|
||||||
|
<span v-for="n in kindPreview.sample_names" :key="n" class="fc-name">
|
||||||
|
{{ n }}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<v-btn
|
||||||
|
color="error" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-delete-sweep"
|
||||||
|
:disabled="!kindPreview.count"
|
||||||
|
:loading="kindCommitting"
|
||||||
|
@click="onKindCommit"
|
||||||
|
>Delete {{ kindPreview.count }} legacy tag(s)</v-btn>
|
||||||
|
</div>
|
||||||
</v-card-text>
|
</v-card-text>
|
||||||
</v-card>
|
</v-card>
|
||||||
</template>
|
</template>
|
||||||
@@ -57,6 +102,9 @@ const store = useAdminStore()
|
|||||||
const preview = ref(null)
|
const preview = ref(null)
|
||||||
const loadingPreview = ref(false)
|
const loadingPreview = ref(false)
|
||||||
const committing = ref(false)
|
const committing = ref(false)
|
||||||
|
const kindPreview = ref(null)
|
||||||
|
const loadingKindPreview = ref(false)
|
||||||
|
const kindCommitting = ref(false)
|
||||||
|
|
||||||
async function onPreview() {
|
async function onPreview() {
|
||||||
loadingPreview.value = true
|
loadingPreview.value = true
|
||||||
@@ -76,6 +124,25 @@ async function onCommit() {
|
|||||||
committing.value = false
|
committing.value = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function onKindPreview() {
|
||||||
|
loadingKindPreview.value = true
|
||||||
|
try {
|
||||||
|
kindPreview.value = await store.purgeLegacyTags({ dryRun: true })
|
||||||
|
} finally {
|
||||||
|
loadingKindPreview.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onKindCommit() {
|
||||||
|
kindCommitting.value = true
|
||||||
|
try {
|
||||||
|
await store.purgeLegacyTags({ dryRun: false })
|
||||||
|
kindPreview.value = { count: 0, by_kind: {}, by_prefix: {}, sample_names: [] }
|
||||||
|
} finally {
|
||||||
|
kindCommitting.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
|||||||
@@ -11,20 +11,23 @@
|
|||||||
<v-icon start>mdi-image-refresh</v-icon> Run backfill now
|
<v-icon start>mdi-image-refresh</v-icon> Run backfill now
|
||||||
</v-btn>
|
</v-btn>
|
||||||
<span v-if="done" class="ml-3 text-caption">Enqueued.</span>
|
<span v-if="done" class="ml-3 text-caption">Enqueued.</span>
|
||||||
|
<QueueStatusBar queue="thumbnail" queue-label="Thumbnail" />
|
||||||
</v-card-text>
|
</v-card-text>
|
||||||
</v-card>
|
</v-card>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
import { ref } from 'vue'
|
import { ref } from 'vue'
|
||||||
import { useThumbnailsStore } from '../../stores/thumbnails.js'
|
import { useThumbnailsStore } from '../../stores/thumbnails.js'
|
||||||
|
import QueueStatusBar from './QueueStatusBar.vue'
|
||||||
const store = useThumbnailsStore()
|
const store = useThumbnailsStore()
|
||||||
const busy = ref(false)
|
const busy = ref(false)
|
||||||
const done = ref(false)
|
const done = ref(false)
|
||||||
async function run () {
|
async function run () {
|
||||||
busy.value = true
|
busy.value = true
|
||||||
try { await store.triggerBackfill(); done.value = true }
|
try { await store.triggerBackfill(); done.value = true }
|
||||||
catch (e) { window.__fcToast?.({ text: e.message, type: 'error' }) }
|
catch (e) { toast({ text: e.message, type: 'error' }) }
|
||||||
finally { busy.value = false }
|
finally { busy.value = false }
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -0,0 +1,133 @@
|
|||||||
|
<template>
|
||||||
|
<v-card v-if="active.length" variant="tonal" color="info" class="fc-active mb-4">
|
||||||
|
<div class="fc-active__head">
|
||||||
|
<span class="fc-active__pulse" />
|
||||||
|
<span class="fc-active__title">
|
||||||
|
<template v-if="running.length">
|
||||||
|
{{ running.length }} downloading
|
||||||
|
</template>
|
||||||
|
<template v-if="running.length && queued.length"> · </template>
|
||||||
|
<template v-if="queued.length">
|
||||||
|
{{ queued.length }} queued
|
||||||
|
</template>
|
||||||
|
</span>
|
||||||
|
<v-spacer />
|
||||||
|
<span class="fc-active__live">live</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="fc-active__body">
|
||||||
|
<div
|
||||||
|
v-for="e in running" :key="e.id"
|
||||||
|
class="fc-active__row fc-active__row--running"
|
||||||
|
>
|
||||||
|
<span class="fc-active__dot" />
|
||||||
|
<PlatformChip :platform="e.platform" size="x-small" />
|
||||||
|
<span class="fc-active__artist">{{ e.artist_name || '—' }}</span>
|
||||||
|
<v-spacer />
|
||||||
|
<span class="fc-active__timer" :title="`started ${e.started_at}`">
|
||||||
|
{{ elapsed(e.started_at) }}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div
|
||||||
|
v-for="e in queued" :key="e.id"
|
||||||
|
class="fc-active__row fc-active__row--queued"
|
||||||
|
>
|
||||||
|
<v-icon size="x-small" class="fc-active__queue-icon">mdi-clock-outline</v-icon>
|
||||||
|
<PlatformChip :platform="e.platform" size="x-small" />
|
||||||
|
<span class="fc-active__artist">{{ e.artist_name || '—' }}</span>
|
||||||
|
<v-spacer />
|
||||||
|
<span class="fc-active__queued">queued</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</v-card>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||||
|
|
||||||
|
import { useDownloadsStore } from '../../stores/downloads.js'
|
||||||
|
import PlatformChip from './PlatformChip.vue'
|
||||||
|
|
||||||
|
const store = useDownloadsStore()
|
||||||
|
|
||||||
|
const running = computed(() => store.activeEvents.filter((e) => e.status === 'running'))
|
||||||
|
const queued = computed(() => store.activeEvents.filter((e) => e.status === 'pending'))
|
||||||
|
const active = computed(() => [...running.value, ...queued.value])
|
||||||
|
|
||||||
|
// Ticking clock so the running timers update every second. started_at is
|
||||||
|
// tz-aware UTC; Date parses the instant, so (now - start) is correct in
|
||||||
|
// any viewer timezone.
|
||||||
|
const now = ref(Date.now())
|
||||||
|
let timer = null
|
||||||
|
onMounted(() => { timer = setInterval(() => { now.value = Date.now() }, 1000) })
|
||||||
|
onUnmounted(() => { if (timer) clearInterval(timer) })
|
||||||
|
|
||||||
|
function elapsed (startedIso) {
|
||||||
|
if (!startedIso) return '—'
|
||||||
|
const start = new Date(startedIso).getTime()
|
||||||
|
if (isNaN(start)) return '—'
|
||||||
|
const secs = Math.max(0, Math.floor((now.value - start) / 1000))
|
||||||
|
const h = Math.floor(secs / 3600)
|
||||||
|
const m = Math.floor((secs % 3600) / 60)
|
||||||
|
const s = secs % 60
|
||||||
|
const mm = String(m).padStart(2, '0')
|
||||||
|
const ss = String(s).padStart(2, '0')
|
||||||
|
return h > 0 ? `${h}:${mm}:${ss}` : `${m}:${ss}`
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-active { border-radius: 8px; }
|
||||||
|
.fc-active__head {
|
||||||
|
display: flex; align-items: center; gap: 10px;
|
||||||
|
padding: 10px 14px;
|
||||||
|
}
|
||||||
|
.fc-active__title {
|
||||||
|
font-weight: 700;
|
||||||
|
text-transform: uppercase; letter-spacing: 0.04em; font-size: 0.85rem;
|
||||||
|
}
|
||||||
|
.fc-active__pulse {
|
||||||
|
width: 10px; height: 10px; border-radius: 50%; flex: 0 0 auto;
|
||||||
|
background: rgb(var(--v-theme-info));
|
||||||
|
animation: fc-active-pulse 1.4s ease-in-out infinite;
|
||||||
|
}
|
||||||
|
.fc-active__live {
|
||||||
|
font-size: 0.7rem; text-transform: uppercase; letter-spacing: 0.08em;
|
||||||
|
color: rgb(var(--v-theme-info));
|
||||||
|
}
|
||||||
|
.fc-active__body {
|
||||||
|
padding: 0 14px 12px;
|
||||||
|
display: flex; flex-direction: column; gap: 4px;
|
||||||
|
}
|
||||||
|
.fc-active__row {
|
||||||
|
display: flex; align-items: center; gap: 10px;
|
||||||
|
padding: 7px 10px;
|
||||||
|
border-radius: 6px;
|
||||||
|
background: rgb(var(--v-theme-surface) / 0.5);
|
||||||
|
}
|
||||||
|
.fc-active__row--queued { opacity: 0.75; }
|
||||||
|
.fc-active__dot {
|
||||||
|
width: 8px; height: 8px; border-radius: 50%; flex: 0 0 auto;
|
||||||
|
background: rgb(var(--v-theme-info));
|
||||||
|
animation: fc-active-pulse 1.4s ease-in-out infinite;
|
||||||
|
}
|
||||||
|
.fc-active__queue-icon { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
.fc-active__artist { font-weight: 600; }
|
||||||
|
.fc-active__timer {
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
font-weight: 700;
|
||||||
|
color: rgb(var(--v-theme-info));
|
||||||
|
}
|
||||||
|
.fc-active__queued {
|
||||||
|
font-size: 0.78rem; text-transform: uppercase; letter-spacing: 0.05em;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
}
|
||||||
|
@keyframes fc-active-pulse {
|
||||||
|
0%, 100% { opacity: 1; transform: scale(1); }
|
||||||
|
50% { opacity: 0.35; transform: scale(0.7); }
|
||||||
|
}
|
||||||
|
@media (prefers-reduced-motion: reduce) {
|
||||||
|
.fc-active__pulse, .fc-active__dot { animation: none; }
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -20,6 +20,7 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
import { ref, watch } from 'vue'
|
import { ref, watch } from 'vue'
|
||||||
import { useSourcesStore } from '../../stores/sources.js'
|
import { useSourcesStore } from '../../stores/sources.js'
|
||||||
|
|
||||||
@@ -44,7 +45,7 @@ async function submit() {
|
|||||||
busy.value = true
|
busy.value = true
|
||||||
try {
|
try {
|
||||||
const { artist, created } = await store.findOrCreateArtist(name.value.trim())
|
const { artist, created } = await store.findOrCreateArtist(name.value.trim())
|
||||||
globalThis.window?.__fcToast?.({
|
toast({
|
||||||
text: created ? 'Artist created' : 'Artist already exists',
|
text: created ? 'Artist created' : 'Artist already exists',
|
||||||
type: 'success',
|
type: 'success',
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -24,9 +24,17 @@
|
|||||||
</v-icon>
|
</v-icon>
|
||||||
<span>Expires {{ fmtDate(credential.expires_at) }}</span>
|
<span>Expires {{ fmtDate(credential.expires_at) }}</span>
|
||||||
</div>
|
</div>
|
||||||
<div v-if="credential.last_verified_at" class="fc-cred-card__row">
|
<div v-if="credential.last_verified" class="fc-cred-card__row">
|
||||||
<v-icon size="small" color="on-surface-variant">mdi-shield-check</v-icon>
|
<v-icon size="small" :color="verifyStale ? 'warning' : 'success'">
|
||||||
<span>Last verified {{ fmtDate(credential.last_verified_at) }}</span>
|
{{ verifyStale ? 'mdi-shield-alert' : 'mdi-shield-check' }}
|
||||||
|
</v-icon>
|
||||||
|
<span :class="verifyStale ? 'text-warning' : undefined">
|
||||||
|
Last verified {{ verifiedRelative }}<template v-if="verifyStale"> — re-verify recommended</template>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div v-else-if="!freshlyVerified" class="fc-cred-card__row">
|
||||||
|
<v-icon size="small" color="warning">mdi-shield-alert</v-icon>
|
||||||
|
<span class="text-warning">Never verified — click Verify to confirm</span>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
<template v-else>
|
<template v-else>
|
||||||
@@ -55,7 +63,21 @@
|
|||||||
</v-card-text>
|
</v-card-text>
|
||||||
|
|
||||||
<v-card-actions class="px-3 pb-3 pt-0">
|
<v-card-actions class="px-3 pb-3 pt-0">
|
||||||
<v-spacer />
|
<v-chip
|
||||||
|
v-if="verifyResult"
|
||||||
|
:color="verifyResult.valid === true ? 'success' : (verifyResult.valid === false ? 'error' : 'grey')"
|
||||||
|
size="x-small" variant="tonal" class="me-auto"
|
||||||
|
:title="verifyResult.reason"
|
||||||
|
>{{ verifyChipLabel }}</v-chip>
|
||||||
|
<v-spacer v-else />
|
||||||
|
<v-btn
|
||||||
|
v-if="hasCredential"
|
||||||
|
size="small" variant="text"
|
||||||
|
:loading="verifying"
|
||||||
|
@click="onVerify"
|
||||||
|
>
|
||||||
|
Verify
|
||||||
|
</v-btn>
|
||||||
<v-btn
|
<v-btn
|
||||||
v-if="hasCredential"
|
v-if="hasCredential"
|
||||||
size="small" variant="text" color="error"
|
size="small" variant="text" color="error"
|
||||||
@@ -76,14 +98,45 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed } from 'vue'
|
import { toast } from '../../utils/toast.js'
|
||||||
|
import { computed, ref } from 'vue'
|
||||||
import PlatformChip from './PlatformChip.vue'
|
import PlatformChip from './PlatformChip.vue'
|
||||||
|
import { useCredentialsStore } from '../../stores/credentials.js'
|
||||||
|
import { formatLocalDate } from '../../utils/date.js'
|
||||||
|
|
||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
platform: { type: Object, required: true },
|
platform: { type: Object, required: true },
|
||||||
credential: { type: Object, default: null },
|
credential: { type: Object, default: null },
|
||||||
})
|
})
|
||||||
defineEmits(['replace', 'remove'])
|
const emit = defineEmits(['replace', 'remove', 'verified'])
|
||||||
|
|
||||||
|
const credsStore = useCredentialsStore()
|
||||||
|
const verifying = ref(false)
|
||||||
|
const verifyResult = ref(null)
|
||||||
|
|
||||||
|
const verifyChipLabel = computed(() => {
|
||||||
|
if (!verifyResult.value) return ''
|
||||||
|
if (verifyResult.value.valid === true) return 'Verified ✓'
|
||||||
|
if (verifyResult.value.valid === false) return 'Failed'
|
||||||
|
return 'Untestable'
|
||||||
|
})
|
||||||
|
|
||||||
|
async function onVerify() {
|
||||||
|
verifying.value = true
|
||||||
|
verifyResult.value = null
|
||||||
|
try {
|
||||||
|
const res = await credsStore.verify(props.platform.key)
|
||||||
|
verifyResult.value = res
|
||||||
|
const type = res.valid === true ? 'success' : (res.valid === false ? 'error' : 'info')
|
||||||
|
toast({ text: `${props.platform.name}: ${res.reason}`, type })
|
||||||
|
if (res.valid === true) emit('verified')
|
||||||
|
} catch (e) {
|
||||||
|
verifyResult.value = { valid: false, reason: e.message }
|
||||||
|
toast({ text: `Verify failed: ${e.message}`, type: 'error' })
|
||||||
|
} finally {
|
||||||
|
verifying.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const hasCredential = computed(() => !!props.credential)
|
const hasCredential = computed(() => !!props.credential)
|
||||||
|
|
||||||
@@ -96,14 +149,42 @@ const expiringSoon = computed(() => {
|
|||||||
return diff > 0 && diff < 7
|
return diff > 0 && diff < 7
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Verification staleness: a stored cookie/token that hasn't been confirmed
|
||||||
|
// in a month is the silent-failure trap (subscribestar/HF). Nudge the
|
||||||
|
// operator to re-verify before a scheduled download dies on bad auth.
|
||||||
|
const STALE_DAYS = 30
|
||||||
|
// A successful Verify in this session clears the warning even before the
|
||||||
|
// parent reloads the credential list (last_verified prop is still old).
|
||||||
|
const freshlyVerified = computed(() => verifyResult.value?.valid === true)
|
||||||
|
const verifiedAgeDays = computed(() => {
|
||||||
|
const v = props.credential?.last_verified
|
||||||
|
if (!v) return null
|
||||||
|
return (Date.now() - new Date(v).getTime()) / 86400_000
|
||||||
|
})
|
||||||
|
const verifyStale = computed(() => {
|
||||||
|
if (!hasCredential.value || freshlyVerified.value) return false
|
||||||
|
const age = verifiedAgeDays.value
|
||||||
|
return age == null || age > STALE_DAYS
|
||||||
|
})
|
||||||
|
const verifiedRelative = computed(() => {
|
||||||
|
const age = verifiedAgeDays.value
|
||||||
|
if (age == null) return '—'
|
||||||
|
if (age < 1) return 'today'
|
||||||
|
if (age < 2) return 'yesterday'
|
||||||
|
if (age < 30) return `${Math.floor(age)}d ago`
|
||||||
|
if (age < 365) return `${Math.floor(age / 30)}mo ago`
|
||||||
|
return `${Math.floor(age / 365)}y ago`
|
||||||
|
})
|
||||||
|
|
||||||
const statusLabel = computed(() => {
|
const statusLabel = computed(() => {
|
||||||
if (!hasCredential.value) return 'Not configured'
|
if (!hasCredential.value) return 'Not configured'
|
||||||
if (expiringSoon.value) return 'Expiring soon'
|
if (expiringSoon.value) return 'Expiring soon'
|
||||||
|
if (verifyStale.value) return 'Verify due'
|
||||||
return 'Active'
|
return 'Active'
|
||||||
})
|
})
|
||||||
const statusColor = computed(() => {
|
const statusColor = computed(() => {
|
||||||
if (!hasCredential.value) return 'grey'
|
if (!hasCredential.value) return 'grey'
|
||||||
if (expiringSoon.value) return 'warning'
|
if (expiringSoon.value || verifyStale.value) return 'warning'
|
||||||
return 'success'
|
return 'success'
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -115,8 +196,7 @@ const howToFallback = computed(() => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
function fmtDate(iso) {
|
function fmtDate(iso) {
|
||||||
if (!iso) return '—'
|
return iso ? formatLocalDate(iso) : '—'
|
||||||
return iso.slice(0, 10)
|
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
<template>
|
||||||
|
<div class="fc-spark" :title="summaryTip">
|
||||||
|
<div class="fc-spark__bars">
|
||||||
|
<div
|
||||||
|
v-for="(b, i) in buckets" :key="i"
|
||||||
|
class="fc-spark__bar"
|
||||||
|
:title="barTip(b)"
|
||||||
|
>
|
||||||
|
<div class="fc-spark__fill" :style="{ height: fillPct(b) + '%' }">
|
||||||
|
<div
|
||||||
|
v-if="b.error > 0"
|
||||||
|
class="fc-spark__err"
|
||||||
|
:style="{ height: errPct(b) + '%' }"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="fc-spark__caption">
|
||||||
|
{{ hours }}h · {{ totalEvents }} events
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed } from 'vue'
|
||||||
|
|
||||||
|
const props = defineProps({
|
||||||
|
// [{ hour, ok, error, other, total }] oldest-first
|
||||||
|
buckets: { type: Array, default: () => [] },
|
||||||
|
hours: { type: Number, default: 24 },
|
||||||
|
})
|
||||||
|
|
||||||
|
const maxTotal = computed(() =>
|
||||||
|
Math.max(1, ...props.buckets.map((b) => b.total || 0)),
|
||||||
|
)
|
||||||
|
const totalEvents = computed(() =>
|
||||||
|
props.buckets.reduce((n, b) => n + (b.total || 0), 0),
|
||||||
|
)
|
||||||
|
|
||||||
|
function fillPct(b) {
|
||||||
|
return Math.round(((b.total || 0) / maxTotal.value) * 100)
|
||||||
|
}
|
||||||
|
function errPct(b) {
|
||||||
|
if (!b.total) return 0
|
||||||
|
return Math.round((b.error / b.total) * 100)
|
||||||
|
}
|
||||||
|
function barTip(b) {
|
||||||
|
const t = new Date(b.hour)
|
||||||
|
const label = t.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })
|
||||||
|
return `${label} — ${b.total} events (${b.ok} ok, ${b.error} failed)`
|
||||||
|
}
|
||||||
|
const summaryTip = computed(
|
||||||
|
() => `Download activity, last ${props.hours}h (red = failures)`,
|
||||||
|
)
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-spark {
|
||||||
|
display: flex; flex-direction: column; gap: 2px;
|
||||||
|
min-width: 160px;
|
||||||
|
}
|
||||||
|
.fc-spark__bars {
|
||||||
|
display: flex; align-items: flex-end; gap: 2px;
|
||||||
|
height: 36px;
|
||||||
|
}
|
||||||
|
.fc-spark__bar {
|
||||||
|
flex: 1 1 0;
|
||||||
|
height: 100%;
|
||||||
|
display: flex; align-items: flex-end;
|
||||||
|
min-width: 2px;
|
||||||
|
}
|
||||||
|
.fc-spark__fill {
|
||||||
|
width: 100%;
|
||||||
|
min-height: 1px;
|
||||||
|
position: relative;
|
||||||
|
border-radius: 2px 2px 0 0;
|
||||||
|
background: rgb(var(--v-theme-accent) / 0.7);
|
||||||
|
}
|
||||||
|
.fc-spark__err {
|
||||||
|
position: absolute; bottom: 0; left: 0;
|
||||||
|
width: 100%;
|
||||||
|
border-radius: 0 0 2px 2px;
|
||||||
|
background: rgb(var(--v-theme-error));
|
||||||
|
}
|
||||||
|
.fc-spark__caption {
|
||||||
|
font-size: 0.68rem;
|
||||||
|
text-transform: uppercase;
|
||||||
|
letter-spacing: 0.05em;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -1,36 +1,39 @@
|
|||||||
<template>
|
<template>
|
||||||
<div class="fc-dl-stats">
|
<div class="fc-dl-stats">
|
||||||
<v-chip
|
<v-chip
|
||||||
v-for="s in STAT_DEFS" :key="s.key"
|
v-for="s in STAT_DEFS" :key="s.value"
|
||||||
:color="s.color"
|
:color="s.color"
|
||||||
variant="tonal"
|
:variant="activeStatus === s.value ? 'elevated' : 'tonal'"
|
||||||
:prepend-icon="s.icon"
|
:prepend-icon="s.icon"
|
||||||
size="default"
|
size="default"
|
||||||
|
:class="{ 'fc-dl-stats__active': activeStatus === s.value }"
|
||||||
|
@click="$emit('select', activeStatus === s.value ? null : s.value)"
|
||||||
>
|
>
|
||||||
{{ s.label }}
|
{{ s.label }}
|
||||||
<strong class="ms-1">{{ stats[s.key] ?? 0 }}</strong>
|
<strong class="ms-1">{{ stats[s.value] ?? 0 }}</strong>
|
||||||
</v-chip>
|
</v-chip>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { DOWNLOAD_STATUSES as STAT_DEFS } from '../../utils/downloadStatus.js'
|
||||||
|
|
||||||
defineProps({
|
defineProps({
|
||||||
stats: { type: Object, required: true },
|
stats: { type: Object, required: true },
|
||||||
|
// The currently-active status filter value (pending|running|ok|error|
|
||||||
|
// skipped) or null. Highlights the matching chip.
|
||||||
|
activeStatus: { type: String, default: null },
|
||||||
})
|
})
|
||||||
|
defineEmits(['select'])
|
||||||
// status keys come straight from the backend ENUM
|
|
||||||
// (pending|running|ok|error|skipped); display order + icons are UI-only.
|
|
||||||
const STAT_DEFS = [
|
|
||||||
{ key: 'pending', label: 'Queued', color: 'grey', icon: 'mdi-clock-outline' },
|
|
||||||
{ key: 'running', label: 'Running', color: 'info', icon: 'mdi-progress-clock' },
|
|
||||||
{ key: 'ok', label: 'Completed', color: 'success', icon: 'mdi-check-circle' },
|
|
||||||
{ key: 'error', label: 'Failed', color: 'error', icon: 'mdi-alert-circle' },
|
|
||||||
{ key: 'skipped', label: 'Skipped', color: 'warning', icon: 'mdi-skip-next' },
|
|
||||||
]
|
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
.fc-dl-stats {
|
.fc-dl-stats {
|
||||||
display: flex; gap: 8px; flex-wrap: wrap;
|
display: flex; gap: 8px; flex-wrap: wrap;
|
||||||
}
|
}
|
||||||
|
.fc-dl-stats .v-chip { cursor: pointer; }
|
||||||
|
.fc-dl-stats__active {
|
||||||
|
outline: 2px solid rgb(var(--v-theme-on-surface) / 0.5);
|
||||||
|
outline-offset: 1px;
|
||||||
|
}
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user