Merge pull request 'Modal suggestions scroll-cap + Celery broker resilience' (#129) from dev into main
Build images / sign-extension (push) Successful in 3s
CI / lint (push) Successful in 3s
CI / frontend-build (push) Successful in 18s
CI / backend-lint-and-test (push) Successful in 28s
Build images / build-web (push) Successful in 2m3s
Build images / build-ml (push) Failing after 2m34s
CI / integration (push) Successful in 3m30s
Build images / sign-extension (push) Successful in 3s
CI / lint (push) Successful in 3s
CI / frontend-build (push) Successful in 18s
CI / backend-lint-and-test (push) Successful in 28s
Build images / build-web (push) Successful in 2m3s
Build images / build-ml (push) Failing after 2m34s
CI / integration (push) Successful in 3m30s
This commit was merged in pull request #129.
This commit is contained in:
@@ -61,7 +61,33 @@ def make_celery() -> Celery:
|
|||||||
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
||||||
task_acks_late=True,
|
task_acks_late=True,
|
||||||
worker_prefetch_multiplier=1,
|
worker_prefetch_multiplier=1,
|
||||||
|
# Broker resilience (2026-06-24): a swarm overlay-network blip after a
|
||||||
|
# redeploy left Redis healthy but transiently unreachable, and a worker
|
||||||
|
# starting in that window crash-looped on the initial broker connect
|
||||||
|
# (kombu OperationalError) instead of waiting it out — needing a manual
|
||||||
|
# Redis reset to recover. Retry the broker FOREVER (None) on startup and
|
||||||
|
# at runtime so a transient outage self-heals when routing returns,
|
||||||
|
# rather than the worker exiting.
|
||||||
broker_connection_retry_on_startup=True,
|
broker_connection_retry_on_startup=True,
|
||||||
|
broker_connection_retry=True,
|
||||||
|
broker_connection_max_retries=None,
|
||||||
|
# Redis-transport socket options (apply to the BROKER connection): a
|
||||||
|
# short connect timeout + TCP keepalive so a dead/blocked socket is
|
||||||
|
# noticed and retried, and a periodic health check that proactively
|
||||||
|
# reconnects a live worker through a network hiccup.
|
||||||
|
broker_transport_options={
|
||||||
|
"socket_connect_timeout": 5,
|
||||||
|
"socket_timeout": 30,
|
||||||
|
"socket_keepalive": True,
|
||||||
|
"retry_on_timeout": True,
|
||||||
|
"health_check_interval": 30,
|
||||||
|
},
|
||||||
|
# Same hardening for the Redis RESULT backend (separate connection pool).
|
||||||
|
redis_socket_connect_timeout=5,
|
||||||
|
redis_socket_timeout=30,
|
||||||
|
redis_socket_keepalive=True,
|
||||||
|
redis_retry_on_timeout=True,
|
||||||
|
redis_backend_health_check_interval=30,
|
||||||
beat_schedule={
|
beat_schedule={
|
||||||
"recover-interrupted-tasks": {
|
"recover-interrupted-tasks": {
|
||||||
"task": "backend.app.tasks.maintenance.recover_interrupted_tasks",
|
"task": "backend.app.tasks.maintenance.recover_interrupted_tasks",
|
||||||
|
|||||||
@@ -12,7 +12,11 @@
|
|||||||
No suggestions above threshold.
|
No suggestions above threshold.
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<template v-else>
|
<!-- Scroll-capped so a long suggestion set (the General bucket can run to
|
||||||
|
dozens) scrolls WITHIN this box instead of stretching the right rail
|
||||||
|
and shoving the Related strip below the fold. Mirrors the
|
||||||
|
ProvenancePanel cards/attachments treatment. -->
|
||||||
|
<div v-else class="fc-suggestions__list">
|
||||||
<SuggestionsCategoryGroup
|
<SuggestionsCategoryGroup
|
||||||
v-for="cat in peopleCats" :key="cat"
|
v-for="cat in peopleCats" :key="cat"
|
||||||
v-show="store.byCategory[cat] && store.byCategory[cat].length"
|
v-show="store.byCategory[cat] && store.byCategory[cat].length"
|
||||||
@@ -27,7 +31,7 @@
|
|||||||
@accept="onAccept" @alias="onAlias" @remove-alias="onRemoveAlias"
|
@accept="onAccept" @alias="onAlias" @remove-alias="onRemoveAlias"
|
||||||
@dismiss="store.dismiss"
|
@dismiss="store.dismiss"
|
||||||
/>
|
/>
|
||||||
</template>
|
</div>
|
||||||
|
|
||||||
<v-dialog v-model="aliasDialog" max-width="480">
|
<v-dialog v-model="aliasDialog" max-width="480">
|
||||||
<AliasPickerDialog
|
<AliasPickerDialog
|
||||||
@@ -125,6 +129,16 @@ async function onRemoveAlias(s) {
|
|||||||
color: rgb(var(--v-theme-on-surface));
|
color: rgb(var(--v-theme-on-surface));
|
||||||
margin-bottom: 8px;
|
margin-bottom: 8px;
|
||||||
}
|
}
|
||||||
|
.fc-suggestions__list {
|
||||||
|
/* Cap the suggestion run so it scrolls internally rather than pushing the
|
||||||
|
Related strip off the bottom of the rail. Hairline scrollbar matching
|
||||||
|
ProvenancePanel's capped regions. */
|
||||||
|
max-height: 320px;
|
||||||
|
overflow-y: auto;
|
||||||
|
scrollbar-width: thin;
|
||||||
|
scrollbar-color: rgb(var(--v-theme-surface-light)) transparent;
|
||||||
|
padding-right: 4px;
|
||||||
|
}
|
||||||
.fc-suggestions__skeleton { display: flex; flex-direction: column; gap: 8px; }
|
.fc-suggestions__skeleton { display: flex; flex-direction: column; gap: 8px; }
|
||||||
.fc-suggestions__skel-row {
|
.fc-suggestions__skel-row {
|
||||||
height: 18px; border-radius: 4px;
|
height: 18px; border-radius: 4px;
|
||||||
|
|||||||
@@ -22,3 +22,17 @@ def test_ping_returns_pong():
|
|||||||
def test_ping_with_value():
|
def test_ping_with_value():
|
||||||
result = ping.delay("hello")
|
result = ping.delay("hello")
|
||||||
assert result.get(timeout=1) == "hello"
|
assert result.get(timeout=1) == "hello"
|
||||||
|
|
||||||
|
|
||||||
|
def test_broker_resilience_configured():
|
||||||
|
# Workers must ride out a transient broker outage (swarm overlay blip after
|
||||||
|
# a redeploy left Redis briefly unreachable) instead of crash-looping on the
|
||||||
|
# initial connect and needing a manual Redis reset (2026-06-24).
|
||||||
|
conf = celery.conf
|
||||||
|
assert conf.broker_connection_retry_on_startup is True
|
||||||
|
assert conf.broker_connection_max_retries is None # retry forever
|
||||||
|
opts = conf.broker_transport_options
|
||||||
|
assert opts["socket_keepalive"] is True
|
||||||
|
assert opts["socket_connect_timeout"] == 5
|
||||||
|
assert opts["health_check_interval"] == 30
|
||||||
|
assert conf.redis_retry_on_timeout is True
|
||||||
|
|||||||
Reference in New Issue
Block a user