What you pay for, what Discord drops, and pixiv switched off #251

Merged
bvandeusen merged 37 commits from dev into main 2026-09-13 12:09:28 -04:00
9 changed files with 305 additions and 4 deletions
Showing only changes of commit a708f5e9db - Show all commits
+31
View File
@@ -20,6 +20,9 @@ from sqlalchemy import Select
from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from ..models import Source
from .gallery_dl import ErrorType
async def get_or_create[T]( async def get_or_create[T](
session: AsyncSession, session: AsyncSession,
@@ -50,3 +53,31 @@ async def get_or_create[T](
except IntegrityError: except IntegrityError:
await sp.rollback() await sp.rollback()
return (await session.execute(select_stmt)).scalar_one(), False return (await session.execute(select_stmt)).scalar_one(), False
# --- shared Source health predicates ----------------------------------------
#
# The subscriptions rollup, the front-door status ribbon and the list endpoint
# all have to agree on what "failing" and "no access" MEAN, or the ribbon says
# 3 and the card it links to shows 4. Same reasoning as get_or_create above:
# divergent copies of one predicate are how the drift creeps in. Defined here
# rather than in source_service because scheduler_service needs them too, and
# source_service already imports scheduler_service (the other direction would
# be a cycle).
def failing_sources_clause():
"""A source is FAILING when its runs are actually erroring.
Deliberately not `last_error IS NOT NULL` — a tier-limited source clears
last_error and keeps a chip, and must never be counted as broken.
"""
return Source.consecutive_failures > 0
def no_access_sources_clause():
"""A source we can't see the content of: the walk works, the tier doesn't
grant it (#874 / milestone #387 phase A). Not a failure — kept separate
from failing_sources_clause on purpose, and the two are disjoint because
an informational class only ever rides an otherwise-OK run."""
return Source.error_type == ErrorType.TIER_LIMITED
+20 -1
View File
@@ -8,12 +8,13 @@ from __future__ import annotations
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
from sqlalchemy import select from sqlalchemy import func, select
from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload from sqlalchemy.orm import selectinload
from ..models import AppSetting, Artist, ImportSettings, Source from ..models import AppSetting, Artist, ImportSettings, Source
from .db_helpers import failing_sources_clause, no_access_sources_clause
MIN_INTERVAL_SECONDS = 60 MIN_INTERVAL_SECONDS = 60
MAX_INTERVAL_SECONDS = 86400 MAX_INTERVAL_SECONDS = 86400
@@ -219,10 +220,28 @@ async def scheduler_status(session: AsyncSession) -> dict:
cooldowns = await active_platform_cooldowns(session) cooldowns = await active_platform_cooldowns(session)
# Ingestion health for the front-door ribbon (#387 B3). Counted over ENABLED
# sources rather than the auto_check subset walked above: a source that is
# erroring or paywalled is worth surfacing whether or not a schedule happens
# to poll it. Two scalar COUNTs, not a second pass over `rows`.
#
# Both predicates are the shared ones, so the ribbon and the surfaces it
# links to cannot disagree about what they are counting.
failing_sources = (await session.execute(
select(func.count()).select_from(Source)
.where(Source.enabled.is_(True), failing_sources_clause())
)).scalar_one()
no_access_sources = (await session.execute(
select(func.count()).select_from(Source)
.where(Source.enabled.is_(True), no_access_sources_clause())
)).scalar_one()
return { return {
"last_tick_at": last_tick_at, "last_tick_at": last_tick_at,
"next_due_at": next_due_at.isoformat() if next_due_at else None, "next_due_at": next_due_at.isoformat() if next_due_at else None,
"due_now": due_now, "due_now": due_now,
"auto_sources": len(rows), "auto_sources": len(rows),
"failing_sources": failing_sources,
"no_access_sources": no_access_sources,
"platform_cooldowns": {p: dt.isoformat() for p, dt in cooldowns.items()}, "platform_cooldowns": {p: dt.isoformat() for p, dt in cooldowns.items()},
} }
+4 -1
View File
@@ -17,6 +17,7 @@ from ..models import (
Post, Post,
Source, Source,
) )
from .db_helpers import failing_sources_clause
from .gallery_dl import ErrorType from .gallery_dl import ErrorType
from .platforms import known_platform_keys from .platforms import known_platform_keys
from .scheduler_service import compute_next_check_at from .scheduler_service import compute_next_check_at
@@ -250,7 +251,9 @@ class SourceService:
stmt = stmt.where(~Source.url.like("sidecar:%")) stmt = stmt.where(~Source.url.like("sidecar:%"))
if failing: if failing:
# Worst-first so the rollup card surfaces the loudest failures. # Worst-first so the rollup card surfaces the loudest failures.
stmt = stmt.where(Source.consecutive_failures > 0).order_by( # Shared clause: the front-door ribbon counts with the same one, so
# it can never report a number this list then contradicts.
stmt = stmt.where(failing_sources_clause()).order_by(
Source.consecutive_failures.desc(), Artist.name.asc(), Source.consecutive_failures.desc(), Artist.name.asc(),
) )
else: else:
@@ -0,0 +1,80 @@
<template>
<!-- Renders nothing at all until it has something true to say. A ribbon that
shows a skeleton or an error on the front door would make the app look
broken every cold load; the feed is the point, this is an aside. -->
<div v-if="status" class="fc-ribbon">
<span class="fc-ribbon__item">
<v-icon size="x-small">mdi-clock-outline</v-icon>
Checked {{ lastCheckedLabel }}
</span>
<RouterLink
v-if="failing" class="fc-ribbon__item fc-ribbon__item--err"
:to="{ path: '/subscriptions', query: { status: 'errors' } }"
>
<v-icon size="x-small">mdi-alert-circle-outline</v-icon>
{{ failing }} {{ failing === 1 ? 'source is' : 'sources are' }} failing
</RouterLink>
<!-- The reason this ribbon exists. Phase A made "we can't see this
creator's posts" a durable fact; without a line here it stays buried
three clicks into Subscriptions, which is exactly where nobody looks
until they already suspect something. -->
<RouterLink
v-if="noAccess" class="fc-ribbon__item fc-ribbon__item--gated"
:to="{ path: '/subscriptions', query: { status: 'no_access' } }"
>
<v-icon size="x-small">mdi-lock-outline</v-icon>
{{ noAccess }} you can't see
</RouterLink>
</div>
</template>
<script setup>
import { computed, onMounted } from 'vue'
import { storeToRefs } from 'pinia'
import { useSourcesStore } from '../../stores/sources.js'
import { formatRelative } from '../../utils/date.js'
const store = useSourcesStore()
const { scheduleStatus: status } = storeToRefs(store)
const failing = computed(() => status.value?.failing_sources || 0)
const noAccess = computed(() => status.value?.no_access_sources || 0)
const lastCheckedLabel = computed(() =>
formatRelative(status.value?.last_tick_at, { nullText: 'never' }),
)
onMounted(() => {
// Swallowed on purpose: this is an aside on the front door, and the feed
// must render whether or not the status call succeeds (rule #164 — a
// genuinely-external-ish fact degrades to absent, it never gates the page).
store.loadScheduleStatus().catch(() => {})
})
</script>
<style scoped>
.fc-ribbon {
display: flex;
align-items: center;
flex-wrap: wrap;
gap: 4px 14px;
padding: 2px 0 10px;
font-size: 0.78rem;
color: rgb(var(--v-theme-on-surface-variant));
}
.fc-ribbon__item {
display: inline-flex;
align-items: center;
gap: 4px;
color: inherit;
text-decoration: none;
}
/* Only the actionable items take a colour, so a healthy instance reads as one
quiet grey line rather than a status dashboard. */
.fc-ribbon__item--err { color: rgb(var(--v-theme-error)); }
.fc-ribbon__item--gated { color: rgb(var(--v-theme-info)); }
.fc-ribbon__item--err:hover,
.fc-ribbon__item--gated:hover { text-decoration: underline; }
</style>
@@ -368,7 +368,19 @@ const platformsStore = usePlatformsStore()
const importStore = useImportStore() const importStore = useImportStore()
const search = ref('') const search = ref('')
const statusFilter = ref('all') // URL-addressable (#387 B3) so the front-door status ribbon can link straight
// to "the sources this number is about" — a count that lands you on an
// unfiltered list makes the reader do the filtering the ribbon just did.
// Mirrors how artistFilter already reads from route.query below.
const statusFilter = computed({
get: () => route.query.status || 'all',
set: (v) => {
const q = { ...route.query }
if (!v || v === 'all') delete q.status
else q.status = v
router.replace({ query: q })
},
})
const needsAttention = ref(false) const needsAttention = ref(false)
const expanded = ref([]) const expanded = ref([])
const selected = ref([]) const selected = ref([])
+4 -1
View File
@@ -35,7 +35,10 @@ const routes = [
// //
// No stickyChrome: unlike Browse/Gallery/Settings this view has no sticky // No stickyChrome: unlike Browse/Gallery/Settings this view has no sticky
// sub-header for the nav to butt against, so the nav keeps its normal fade. // sub-header for the nav to butt against, so the nav keeps its normal fade.
{ path: '/latest', name: 'latest', component: PostsView, meta: { title: 'Latest', navOrder: 5 } }, // `props` turns on the ingestion-status ribbon (#387 B3). Only here: inside
// Browse's Posts tab the same view renders without it.
{ path: '/latest', name: 'latest', component: PostsView, props: { statusRibbon: true },
meta: { title: 'Latest', navOrder: 5 } },
// FC-2: image backbone // FC-2: image backbone
{ path: '/showcase', name: 'showcase', component: ShowcaseView, meta: { title: 'Showcase', navOrder: 10 } }, { path: '/showcase', name: 'showcase', component: ShowcaseView, meta: { title: 'Showcase', navOrder: 10 } },
+12
View File
@@ -49,6 +49,8 @@
<!-- Normal feed --> <!-- Normal feed -->
<template v-else> <template v-else>
<FeedStatusRibbon v-if="statusRibbon" />
<PostsFilterBar <PostsFilterBar
:artist-id="artistFilter" :artist-id="artistFilter"
:platform="platformFilter" :platform="platformFilter"
@@ -89,6 +91,16 @@ import { useRoute, useRouter } from 'vue-router'
import { usePostsStore } from '../stores/posts.js' import { usePostsStore } from '../stores/posts.js'
import PostsFilterBar from '../components/posts/PostsFilterBar.vue' import PostsFilterBar from '../components/posts/PostsFilterBar.vue'
import PostCard from '../components/posts/PostCard.vue' import PostCard from '../components/posts/PostCard.vue'
import FeedStatusRibbon from '../components/posts/FeedStatusRibbon.vue'
// The ingestion-status ribbon is a FRONT-DOOR concern, not a feed concern —
// inside Browse's Posts tab you are looking FOR something, and the Subscriptions
// hub is a click away. Passed as a route prop rather than sniffed from
// route.name so the view does not have to know what it is mounted as, and so
// the router file states the intent in one place.
defineProps({
statusRibbon: { type: Boolean, default: false },
})
const route = useRoute() const route = useRoute()
const router = useRouter() const router = useRouter()
@@ -0,0 +1,76 @@
// @vitest-environment happy-dom
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import FeedStatusRibbon from '../../src/components/posts/FeedStatusRibbon.vue'
import { useSourcesStore } from '../../src/stores/sources.js'
import { freshPinia, mountComponent } from '../support/mountComponent.js'
// #387 B3. This ribbon is the ONLY place phase A's work reaches someone who
// wasn't already looking for it, so what it does and does not say is the whole
// feature. These pin: it stays silent when it has nothing true to report, it
// keeps "failing" and "no access" as separate claims, and it never renders a
// zero — a "0 failing" on the front door is noise that trains you to ignore
// the line, which is exactly what would hide the real number later.
function mountWith (status) {
const pinia = freshPinia()
useSourcesStore().scheduleStatus = status
return mountComponent(FeedStatusRibbon, { pinia })
}
describe('FeedStatusRibbon', () => {
beforeEach(() => {
// The component fetches on mount and swallows failures by design; stub it
// so the assertions are about the seeded state, not a race with the call.
globalThis.fetch = vi.fn(async () => { throw new Error('offline') })
})
afterEach(() => vi.restoreAllMocks())
it('renders nothing before it has a status', () => {
const w = mountWith(null)
expect(w.find('.fc-ribbon').exists()).toBe(false)
})
it('a healthy instance is one quiet line, with no zeroes', () => {
const w = mountWith({
last_tick_at: new Date().toISOString(), failing_sources: 0, no_access_sources: 0,
})
expect(w.find('.fc-ribbon').exists()).toBe(true)
expect(w.text()).toContain('Checked')
// Assert on the claims, not on the digit — a just-now timestamp can render
// its own "0 minutes ago" and a substring check would catch that instead.
expect(w.text()).not.toContain('failing')
expect(w.text()).not.toContain("can't see")
expect(w.find('.fc-ribbon__item--err').exists()).toBe(false)
expect(w.find('.fc-ribbon__item--gated').exists()).toBe(false)
})
it('reports failing and no-access as two separate claims', () => {
const w = mountWith({
last_tick_at: new Date().toISOString(), failing_sources: 2, no_access_sources: 5,
})
expect(w.find('.fc-ribbon__item--err').text()).toContain('2 sources are failing')
expect(w.find('.fc-ribbon__item--gated').text()).toContain("5 you can't see")
})
it('no-access alone does not light the failing item', () => {
// The whole point of phase A: a paywalled creator is not a broken one.
const w = mountWith({
last_tick_at: new Date().toISOString(), failing_sources: 0, no_access_sources: 3,
})
expect(w.find('.fc-ribbon__item--err').exists()).toBe(false)
expect(w.find('.fc-ribbon__item--gated').exists()).toBe(true)
})
it('says "never" rather than a blank when nothing has ever run', () => {
const w = mountWith({ last_tick_at: null, failing_sources: 0, no_access_sources: 0 })
expect(w.text()).toContain('never')
})
it('singularises one failing source', () => {
const w = mountWith({
last_tick_at: new Date().toISOString(), failing_sources: 1, no_access_sources: 0,
})
expect(w.find('.fc-ribbon__item--err').text()).toContain('1 source is failing')
})
})
+65
View File
@@ -12,6 +12,7 @@ import pytest
from backend.app.models import Artist, ImportSettings, Source from backend.app.models import Artist, ImportSettings, Source
from backend.app.services.scheduler_service import ( from backend.app.services.scheduler_service import (
compute_effective_interval, compute_effective_interval,
scheduler_status,
select_due_sources, select_due_sources,
) )
@@ -296,3 +297,67 @@ async def test_set_platform_cooldown_upserts(db):
) )
)).scalars().all() )).scalars().all()
assert len(rows) == 1 assert len(rows) == 1
# --- scheduler_status ingestion counts (#387 B3) --------------------------
@pytest.mark.asyncio
async def test_status_counts_failing_and_no_access_separately(db):
"""The front-door ribbon's two numbers must not bleed into each other.
A tier-limited source has consecutive_failures == 0 by construction (its
run succeeded), so counting failures by `last_error IS NOT NULL` or by the
presence of an error_type would report it as broken — which is the exact
claim phase A exists to stop the app making.
"""
artist = await _seed_artist(db, name="cnt")
db.add_all([
Source(artist_id=artist.id, platform="patreon", url="https://cnt-broken",
enabled=True, consecutive_failures=3, error_type="auth_error"),
Source(artist_id=artist.id, platform="patreon", url="https://cnt-gated",
enabled=True, consecutive_failures=0, error_type="tier_limited"),
Source(artist_id=artist.id, platform="patreon", url="https://cnt-fine",
enabled=True, consecutive_failures=0),
])
await db.commit()
status = await scheduler_status(db)
assert status["failing_sources"] == 1
assert status["no_access_sources"] == 1
@pytest.mark.asyncio
async def test_status_counts_ignore_disabled_sources(db):
"""Disabling is the operator's way of parking a sub they stopped paying for
(#1285 clears its state on the way). A parked source must not keep a number
lit on the front door."""
artist = await _seed_artist(db, name="cnt-off")
db.add_all([
Source(artist_id=artist.id, platform="patreon", url="https://cnt-off-broken",
enabled=False, consecutive_failures=5),
Source(artist_id=artist.id, platform="patreon", url="https://cnt-off-gated",
enabled=False, error_type="tier_limited"),
])
await db.commit()
status = await scheduler_status(db)
assert status["failing_sources"] == 0
assert status["no_access_sources"] == 0
@pytest.mark.asyncio
async def test_status_counts_are_not_scoped_to_auto_check(db):
"""A source erroring on a manual-only artist is still erroring. The counts
deliberately span all enabled sources, unlike `auto_sources` above which
only describes what is on a schedule."""
artist = await _seed_artist(db, auto=False, name="cnt-manual")
db.add(Source(
artist_id=artist.id, platform="patreon", url="https://cnt-manual",
enabled=True, consecutive_failures=2,
))
await db.commit()
status = await scheduler_status(db)
assert status["failing_sources"] == 1
assert status["auto_sources"] == 0