What you pay for, what Discord drops, and pixiv switched off #251

Merged
bvandeusen merged 37 commits from dev into main 2026-09-13 12:09:28 -04:00
9 changed files with 305 additions and 4 deletions
Showing only changes of commit a708f5e9db - Show all commits
+31
View File
@@ -20,6 +20,9 @@ from sqlalchemy import Select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from ..models import Source
from .gallery_dl import ErrorType
async def get_or_create[T](
session: AsyncSession,
@@ -50,3 +53,31 @@ async def get_or_create[T](
except IntegrityError:
await sp.rollback()
return (await session.execute(select_stmt)).scalar_one(), False
# --- shared Source health predicates ----------------------------------------
#
# The subscriptions rollup, the front-door status ribbon and the list endpoint
# all have to agree on what "failing" and "no access" MEAN, or the ribbon says
# 3 and the card it links to shows 4. Same reasoning as get_or_create above:
# divergent copies of one predicate are how the drift creeps in. Defined here
# rather than in source_service because scheduler_service needs them too, and
# source_service already imports scheduler_service (the other direction would
# be a cycle).
def failing_sources_clause():
"""A source is FAILING when its runs are actually erroring.
Deliberately not `last_error IS NOT NULL` — a tier-limited source clears
last_error and keeps a chip, and must never be counted as broken.
"""
return Source.consecutive_failures > 0
def no_access_sources_clause():
"""A source we can't see the content of: the walk works, the tier doesn't
grant it (#874 / milestone #387 phase A). Not a failure — kept separate
from failing_sources_clause on purpose, and the two are disjoint because
an informational class only ever rides an otherwise-OK run."""
return Source.error_type == ErrorType.TIER_LIMITED
+20 -1
View File
@@ -8,12 +8,13 @@ from __future__ import annotations
from datetime import UTC, datetime, timedelta
from sqlalchemy import select
from sqlalchemy import func, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
from ..models import AppSetting, Artist, ImportSettings, Source
from .db_helpers import failing_sources_clause, no_access_sources_clause
MIN_INTERVAL_SECONDS = 60
MAX_INTERVAL_SECONDS = 86400
@@ -219,10 +220,28 @@ async def scheduler_status(session: AsyncSession) -> dict:
cooldowns = await active_platform_cooldowns(session)
# Ingestion health for the front-door ribbon (#387 B3). Counted over ENABLED
# sources rather than the auto_check subset walked above: a source that is
# erroring or paywalled is worth surfacing whether or not a schedule happens
# to poll it. Two scalar COUNTs, not a second pass over `rows`.
#
# Both predicates are the shared ones, so the ribbon and the surfaces it
# links to cannot disagree about what they are counting.
failing_sources = (await session.execute(
select(func.count()).select_from(Source)
.where(Source.enabled.is_(True), failing_sources_clause())
)).scalar_one()
no_access_sources = (await session.execute(
select(func.count()).select_from(Source)
.where(Source.enabled.is_(True), no_access_sources_clause())
)).scalar_one()
return {
"last_tick_at": last_tick_at,
"next_due_at": next_due_at.isoformat() if next_due_at else None,
"due_now": due_now,
"auto_sources": len(rows),
"failing_sources": failing_sources,
"no_access_sources": no_access_sources,
"platform_cooldowns": {p: dt.isoformat() for p, dt in cooldowns.items()},
}
+4 -1
View File
@@ -17,6 +17,7 @@ from ..models import (
Post,
Source,
)
from .db_helpers import failing_sources_clause
from .gallery_dl import ErrorType
from .platforms import known_platform_keys
from .scheduler_service import compute_next_check_at
@@ -250,7 +251,9 @@ class SourceService:
stmt = stmt.where(~Source.url.like("sidecar:%"))
if failing:
# Worst-first so the rollup card surfaces the loudest failures.
stmt = stmt.where(Source.consecutive_failures > 0).order_by(
# Shared clause: the front-door ribbon counts with the same one, so
# it can never report a number this list then contradicts.
stmt = stmt.where(failing_sources_clause()).order_by(
Source.consecutive_failures.desc(), Artist.name.asc(),
)
else:
@@ -0,0 +1,80 @@
<template>
<!-- Renders nothing at all until it has something true to say. A ribbon that
shows a skeleton or an error on the front door would make the app look
broken every cold load; the feed is the point, this is an aside. -->
<div v-if="status" class="fc-ribbon">
<span class="fc-ribbon__item">
<v-icon size="x-small">mdi-clock-outline</v-icon>
Checked {{ lastCheckedLabel }}
</span>
<RouterLink
v-if="failing" class="fc-ribbon__item fc-ribbon__item--err"
:to="{ path: '/subscriptions', query: { status: 'errors' } }"
>
<v-icon size="x-small">mdi-alert-circle-outline</v-icon>
{{ failing }} {{ failing === 1 ? 'source is' : 'sources are' }} failing
</RouterLink>
<!-- The reason this ribbon exists. Phase A made "we can't see this
creator's posts" a durable fact; without a line here it stays buried
three clicks into Subscriptions, which is exactly where nobody looks
until they already suspect something. -->
<RouterLink
v-if="noAccess" class="fc-ribbon__item fc-ribbon__item--gated"
:to="{ path: '/subscriptions', query: { status: 'no_access' } }"
>
<v-icon size="x-small">mdi-lock-outline</v-icon>
{{ noAccess }} you can't see
</RouterLink>
</div>
</template>
<script setup>
import { computed, onMounted } from 'vue'
import { storeToRefs } from 'pinia'
import { useSourcesStore } from '../../stores/sources.js'
import { formatRelative } from '../../utils/date.js'
const store = useSourcesStore()
const { scheduleStatus: status } = storeToRefs(store)
const failing = computed(() => status.value?.failing_sources || 0)
const noAccess = computed(() => status.value?.no_access_sources || 0)
const lastCheckedLabel = computed(() =>
formatRelative(status.value?.last_tick_at, { nullText: 'never' }),
)
onMounted(() => {
// Swallowed on purpose: this is an aside on the front door, and the feed
// must render whether or not the status call succeeds (rule #164 — a
// genuinely-external-ish fact degrades to absent, it never gates the page).
store.loadScheduleStatus().catch(() => {})
})
</script>
<style scoped>
.fc-ribbon {
display: flex;
align-items: center;
flex-wrap: wrap;
gap: 4px 14px;
padding: 2px 0 10px;
font-size: 0.78rem;
color: rgb(var(--v-theme-on-surface-variant));
}
.fc-ribbon__item {
display: inline-flex;
align-items: center;
gap: 4px;
color: inherit;
text-decoration: none;
}
/* Only the actionable items take a colour, so a healthy instance reads as one
quiet grey line rather than a status dashboard. */
.fc-ribbon__item--err { color: rgb(var(--v-theme-error)); }
.fc-ribbon__item--gated { color: rgb(var(--v-theme-info)); }
.fc-ribbon__item--err:hover,
.fc-ribbon__item--gated:hover { text-decoration: underline; }
</style>
@@ -368,7 +368,19 @@ const platformsStore = usePlatformsStore()
const importStore = useImportStore()
const search = ref('')
const statusFilter = ref('all')
// URL-addressable (#387 B3) so the front-door status ribbon can link straight
// to "the sources this number is about" — a count that lands you on an
// unfiltered list makes the reader do the filtering the ribbon just did.
// Mirrors how artistFilter already reads from route.query below.
const statusFilter = computed({
get: () => route.query.status || 'all',
set: (v) => {
const q = { ...route.query }
if (!v || v === 'all') delete q.status
else q.status = v
router.replace({ query: q })
},
})
const needsAttention = ref(false)
const expanded = ref([])
const selected = ref([])
+4 -1
View File
@@ -35,7 +35,10 @@ const routes = [
//
// No stickyChrome: unlike Browse/Gallery/Settings this view has no sticky
// sub-header for the nav to butt against, so the nav keeps its normal fade.
{ path: '/latest', name: 'latest', component: PostsView, meta: { title: 'Latest', navOrder: 5 } },
// `props` turns on the ingestion-status ribbon (#387 B3). Only here: inside
// Browse's Posts tab the same view renders without it.
{ path: '/latest', name: 'latest', component: PostsView, props: { statusRibbon: true },
meta: { title: 'Latest', navOrder: 5 } },
// FC-2: image backbone
{ path: '/showcase', name: 'showcase', component: ShowcaseView, meta: { title: 'Showcase', navOrder: 10 } },
+12
View File
@@ -49,6 +49,8 @@
<!-- Normal feed -->
<template v-else>
<FeedStatusRibbon v-if="statusRibbon" />
<PostsFilterBar
:artist-id="artistFilter"
:platform="platformFilter"
@@ -89,6 +91,16 @@ import { useRoute, useRouter } from 'vue-router'
import { usePostsStore } from '../stores/posts.js'
import PostsFilterBar from '../components/posts/PostsFilterBar.vue'
import PostCard from '../components/posts/PostCard.vue'
import FeedStatusRibbon from '../components/posts/FeedStatusRibbon.vue'
// The ingestion-status ribbon is a FRONT-DOOR concern, not a feed concern —
// inside Browse's Posts tab you are looking FOR something, and the Subscriptions
// hub is a click away. Passed as a route prop rather than sniffed from
// route.name so the view does not have to know what it is mounted as, and so
// the router file states the intent in one place.
defineProps({
statusRibbon: { type: Boolean, default: false },
})
const route = useRoute()
const router = useRouter()
@@ -0,0 +1,76 @@
// @vitest-environment happy-dom
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import FeedStatusRibbon from '../../src/components/posts/FeedStatusRibbon.vue'
import { useSourcesStore } from '../../src/stores/sources.js'
import { freshPinia, mountComponent } from '../support/mountComponent.js'
// #387 B3. This ribbon is the ONLY place phase A's work reaches someone who
// wasn't already looking for it, so what it does and does not say is the whole
// feature. These pin: it stays silent when it has nothing true to report, it
// keeps "failing" and "no access" as separate claims, and it never renders a
// zero — a "0 failing" on the front door is noise that trains you to ignore
// the line, which is exactly what would hide the real number later.
function mountWith (status) {
const pinia = freshPinia()
useSourcesStore().scheduleStatus = status
return mountComponent(FeedStatusRibbon, { pinia })
}
describe('FeedStatusRibbon', () => {
beforeEach(() => {
// The component fetches on mount and swallows failures by design; stub it
// so the assertions are about the seeded state, not a race with the call.
globalThis.fetch = vi.fn(async () => { throw new Error('offline') })
})
afterEach(() => vi.restoreAllMocks())
it('renders nothing before it has a status', () => {
const w = mountWith(null)
expect(w.find('.fc-ribbon').exists()).toBe(false)
})
it('a healthy instance is one quiet line, with no zeroes', () => {
const w = mountWith({
last_tick_at: new Date().toISOString(), failing_sources: 0, no_access_sources: 0,
})
expect(w.find('.fc-ribbon').exists()).toBe(true)
expect(w.text()).toContain('Checked')
// Assert on the claims, not on the digit — a just-now timestamp can render
// its own "0 minutes ago" and a substring check would catch that instead.
expect(w.text()).not.toContain('failing')
expect(w.text()).not.toContain("can't see")
expect(w.find('.fc-ribbon__item--err').exists()).toBe(false)
expect(w.find('.fc-ribbon__item--gated').exists()).toBe(false)
})
it('reports failing and no-access as two separate claims', () => {
const w = mountWith({
last_tick_at: new Date().toISOString(), failing_sources: 2, no_access_sources: 5,
})
expect(w.find('.fc-ribbon__item--err').text()).toContain('2 sources are failing')
expect(w.find('.fc-ribbon__item--gated').text()).toContain("5 you can't see")
})
it('no-access alone does not light the failing item', () => {
// The whole point of phase A: a paywalled creator is not a broken one.
const w = mountWith({
last_tick_at: new Date().toISOString(), failing_sources: 0, no_access_sources: 3,
})
expect(w.find('.fc-ribbon__item--err').exists()).toBe(false)
expect(w.find('.fc-ribbon__item--gated').exists()).toBe(true)
})
it('says "never" rather than a blank when nothing has ever run', () => {
const w = mountWith({ last_tick_at: null, failing_sources: 0, no_access_sources: 0 })
expect(w.text()).toContain('never')
})
it('singularises one failing source', () => {
const w = mountWith({
last_tick_at: new Date().toISOString(), failing_sources: 1, no_access_sources: 0,
})
expect(w.find('.fc-ribbon__item--err').text()).toContain('1 source is failing')
})
})
+65
View File
@@ -12,6 +12,7 @@ import pytest
from backend.app.models import Artist, ImportSettings, Source
from backend.app.services.scheduler_service import (
compute_effective_interval,
scheduler_status,
select_due_sources,
)
@@ -296,3 +297,67 @@ async def test_set_platform_cooldown_upserts(db):
)
)).scalars().all()
assert len(rows) == 1
# --- scheduler_status ingestion counts (#387 B3) --------------------------
@pytest.mark.asyncio
async def test_status_counts_failing_and_no_access_separately(db):
"""The front-door ribbon's two numbers must not bleed into each other.
A tier-limited source has consecutive_failures == 0 by construction (its
run succeeded), so counting failures by `last_error IS NOT NULL` or by the
presence of an error_type would report it as broken — which is the exact
claim phase A exists to stop the app making.
"""
artist = await _seed_artist(db, name="cnt")
db.add_all([
Source(artist_id=artist.id, platform="patreon", url="https://cnt-broken",
enabled=True, consecutive_failures=3, error_type="auth_error"),
Source(artist_id=artist.id, platform="patreon", url="https://cnt-gated",
enabled=True, consecutive_failures=0, error_type="tier_limited"),
Source(artist_id=artist.id, platform="patreon", url="https://cnt-fine",
enabled=True, consecutive_failures=0),
])
await db.commit()
status = await scheduler_status(db)
assert status["failing_sources"] == 1
assert status["no_access_sources"] == 1
@pytest.mark.asyncio
async def test_status_counts_ignore_disabled_sources(db):
"""Disabling is the operator's way of parking a sub they stopped paying for
(#1285 clears its state on the way). A parked source must not keep a number
lit on the front door."""
artist = await _seed_artist(db, name="cnt-off")
db.add_all([
Source(artist_id=artist.id, platform="patreon", url="https://cnt-off-broken",
enabled=False, consecutive_failures=5),
Source(artist_id=artist.id, platform="patreon", url="https://cnt-off-gated",
enabled=False, error_type="tier_limited"),
])
await db.commit()
status = await scheduler_status(db)
assert status["failing_sources"] == 0
assert status["no_access_sources"] == 0
@pytest.mark.asyncio
async def test_status_counts_are_not_scoped_to_auto_check(db):
"""A source erroring on a manual-only artist is still erroring. The counts
deliberately span all enabled sources, unlike `auto_sources` above which
only describes what is on a schedule."""
artist = await _seed_artist(db, auto=False, name="cnt-manual")
db.add(Source(
artist_id=artist.id, platform="patreon", url="https://cnt-manual",
enabled=True, consecutive_failures=2,
))
await db.commit()
status = await scheduler_status(db)
assert status["failing_sources"] == 1
assert status["auto_sources"] == 0