feat(server/diagnostics): device debug-reporting ingest + admin timeline + retention (M9)

New diagnostic_events table + per-account users.debug_mode_enabled flag.
When an account's flag is on, its client(s) POST a batch timeseries of
connectivity / UPnP-sync / power / lifecycle events to /api/diagnostics
(no-op 204 when off, kind whitelist mirrors the CHECK constraint).

Admin surface: GET /api/admin/diagnostics (optional account/device/kind/
time-window filters, RFC3339-or-epoch-ms, export-sized paging) + a
/diagnostics/devices overview + PUT /api/admin/users/{id}/debug-mode to
flip an account remotely while a bug is live. debug_mode_enabled is now
exposed on /api/me (client gate) and the admin user views.

Retention: a 30-day gc-worker sweep (GcPruneDiagnostics), keyed on the
server clock so a skewed device clock can't keep rows alive.

Refs Scribe M9 (#119), tasks #1172 #1173.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01K55iTxn95BtshocgdE1shW

internal/api/admin_users.go

@@ -22,6 +22,7 @@ type adminUserView struct {
 	DisplayName         *string `json:"display_name"`
 	IsAdmin             bool    `json:"is_admin"`
 	AutoApproveRequests bool    `json:"auto_approve_requests"`
+	DebugModeEnabled    bool    `json:"debug_mode_enabled"`
 	CreatedAt           string  `json:"created_at"`
 }
 
@@ -44,6 +45,7 @@ func (h *handlers) handleAdminListUsers(w http.ResponseWriter, r *http.Request)
 			DisplayName:         row.DisplayName,
 			IsAdmin:             row.IsAdmin,
 			AutoApproveRequests: row.AutoApproveRequests,
+			DebugModeEnabled:    row.DebugModeEnabled,
 			CreatedAt:           row.CreatedAt.Time.UTC().Format(time.RFC3339),
 		}
 		out = append(out, v)
@@ -120,6 +122,7 @@ func (h *handlers) handleUpdateUserAdmin(w http.ResponseWriter, r *http.Request)
 		DisplayName:         updated.DisplayName,
 		IsAdmin:             updated.IsAdmin,
 		AutoApproveRequests: updated.AutoApproveRequests,
+		DebugModeEnabled:    updated.DebugModeEnabled,
 		CreatedAt:           updated.CreatedAt.Time.UTC().Format(time.RFC3339),
 	})
 }
@@ -189,6 +192,7 @@ func (h *handlers) handleAdminCreateUser(w http.ResponseWriter, r *http.Request)
 		DisplayName:         user.DisplayName,
 		IsAdmin:             user.IsAdmin,
 		AutoApproveRequests: user.AutoApproveRequests,
+		DebugModeEnabled:    user.DebugModeEnabled,
 		CreatedAt:           user.CreatedAt.Time.UTC().Format(time.RFC3339),
 	})
 }
@@ -329,6 +333,7 @@ func (h *handlers) handleAdminAutoApproveToggle(w http.ResponseWriter, r *http.R
 		DisplayName:         updated.DisplayName,
 		IsAdmin:             updated.IsAdmin,
 		AutoApproveRequests: updated.AutoApproveRequests,
+		DebugModeEnabled:    updated.DebugModeEnabled,
 		CreatedAt:           updated.CreatedAt.Time.UTC().Format(time.RFC3339),
 	})
 }

internal/api/api.go

@@ -135,6 +135,12 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
 			// load failures). Admin-only inbox; any user can report.
 			authed.Post("/playback-errors", h.handleReportPlaybackError)
 
+			// Device diagnostics ingest (M9). Any signed-in user can
+			// POST a batch, but events are only stored when the
+			// account's debug_mode_enabled flag is on (handler no-ops
+			// otherwise). Admin views live under /admin/diagnostics.
+			authed.Post("/diagnostics", h.handleReportDiagnostics)
+
 			// Self-hosted in-app update channel (#397). Auth-gated to
 			// prevent anonymous bandwidth abuse on the APK stream;
 			// /apk additionally per-user rate-limited.
@@ -181,6 +187,11 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
 				admin.Delete("/users/{id}", h.handleAdminDeleteUser)
 				admin.Post("/users/{id}/reset-password", h.handleAdminResetPassword)
 				admin.Put("/users/{id}/auto-approve", h.handleAdminAutoApproveToggle)
+				admin.Put("/users/{id}/debug-mode", h.handleAdminDebugModeToggle)
+
+				// Device diagnostics timeline + device overview (M9).
+				admin.Get("/diagnostics", h.handleListAdminDiagnostics)
+				admin.Get("/diagnostics/devices", h.handleListAdminDiagnosticDevices)
 
 				admin.Get("/cover-sources", h.handleListCoverSources)
 				admin.Patch("/cover-sources/{provider_id}", h.handleUpdateCoverSource)

internal/api/diagnostics.go

@@ -0,0 +1,345 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgtype"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
+)
+
+// validDiagnosticKinds is the COARSE category whitelist. Mirrors the
+// CHECK constraint in migration 0036 — keep both in sync (per the
+// enum-CHECK-whitelist rule). The finer event discriminator + all
+// event-specific fields live inside each event's payload, so new event
+// variants do NOT require a new category here.
+var validDiagnosticKinds = map[string]struct{}{
+	"connectivity": {},
+	"upnp_sync":    {},
+	"power":        {},
+	"lifecycle":    {},
+	"heartbeat":    {},
+	"http":         {},
+}
+
+// maxDiagnosticBatch caps a single ingest call. The client buffers and
+// flushes in batches of ~100; 500 leaves generous headroom while
+// bounding a misbehaving client's per-request work.
+const maxDiagnosticBatch = 500
+
+// Diagnostics list paging. Larger than the generic parsePaging caps
+// because the admin timeline is meant to be exported as a slice and
+// handed off for analysis — a heartbeat-dense window has many rows.
+const (
+	defaultDiagnosticPageSize = 500
+	maxDiagnosticPageSize     = 5000
+)
+
+// diagnosticEventIn is one event in a POST /api/diagnostics batch.
+// OccurredAt is the client device clock as epoch milliseconds (may be
+// skewed — the server also stamps received_at). Payload is opaque JSON
+// carrying the event's sub-type + fields.
+type diagnosticEventIn struct {
+	Kind       string          `json:"kind"`
+	OccurredAt int64           `json:"occurred_at"`
+	Payload    json.RawMessage `json:"payload"`
+}
+
+// reportDiagnosticsRequest is the body of POST /api/diagnostics — one
+// batch from one device.
+type reportDiagnosticsRequest struct {
+	ClientID   string              `json:"client_id"`
+	AppVersion *string             `json:"app_version,omitempty"`
+	OsVersion  *string             `json:"os_version,omitempty"`
+	Events     []diagnosticEventIn `json:"events"`
+}
+
+// handleReportDiagnostics implements POST /api/diagnostics. Any signed-in
+// user can call it, but events are only stored when the account's
+// debug_mode_enabled flag is on; otherwise the call is a 204 no-op so the
+// client can safely drop the batch (it gates locally on /api/me, this is
+// the race-safe backstop for "admin just disabled debug").
+func (h *handlers) handleReportDiagnostics(w http.ResponseWriter, r *http.Request) {
+	user, ok := requireUser(w, r)
+	if !ok {
+		return
+	}
+	if !user.DebugModeEnabled {
+		w.WriteHeader(http.StatusNoContent)
+		return
+	}
+	var req reportDiagnosticsRequest
+	if !decodeBody(w, r, &req) {
+		return
+	}
+	if strings.TrimSpace(req.ClientID) == "" {
+		writeErr(w, apierror.BadRequest("bad_request", "client_id required"))
+		return
+	}
+	if len(req.Events) == 0 {
+		writeJSON(w, http.StatusAccepted, map[string]int{"accepted": 0})
+		return
+	}
+	if len(req.Events) > maxDiagnosticBatch {
+		writeErr(w, apierror.BadRequest("batch_too_large", "too many events in one batch"))
+		return
+	}
+	for i := range req.Events {
+		if _, valid := validDiagnosticKinds[req.Events[i].Kind]; !valid {
+			writeErr(w, apierror.BadRequest("invalid_kind", "unknown diagnostic kind"))
+			return
+		}
+	}
+
+	err := pgx.BeginFunc(r.Context(), h.pool, func(tx pgx.Tx) error {
+		q := dbq.New(tx)
+		for i := range req.Events {
+			ev := req.Events[i]
+			payload := ev.Payload
+			if len(payload) == 0 {
+				payload = json.RawMessage("{}")
+			}
+			if _, err := q.InsertDiagnosticEvent(r.Context(), dbq.InsertDiagnosticEventParams{
+				UserID:     user.ID,
+				ClientID:   req.ClientID,
+				AppVersion: req.AppVersion,
+				OsVersion:  req.OsVersion,
+				Kind:       ev.Kind,
+				Payload:    payload,
+				OccurredAt: pgtype.Timestamptz{Time: time.UnixMilli(ev.OccurredAt).UTC(), Valid: true},
+			}); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		writeErrWithLog(w, h.logger, "diagnostics: insert batch failed", apierror.Internal(err))
+		return
+	}
+	writeJSON(w, http.StatusAccepted, map[string]int{"accepted": len(req.Events)})
+}
+
+// adminDiagnosticView is one row in the admin timeline response. Payload
+// is passed through verbatim as raw JSON so the SPA can render / export
+// the structured event without the server re-shaping it.
+type adminDiagnosticView struct {
+	ID         string          `json:"id"`
+	UserID     string          `json:"user_id"`
+	Username   string          `json:"username"`
+	ClientID   string          `json:"client_id"`
+	AppVersion *string         `json:"app_version,omitempty"`
+	OsVersion  *string         `json:"os_version,omitempty"`
+	Kind       string          `json:"kind"`
+	Payload    json.RawMessage `json:"payload"`
+	OccurredAt string          `json:"occurred_at"`
+	ReceivedAt string          `json:"received_at"`
+}
+
+// handleListAdminDiagnostics implements GET /api/admin/diagnostics with
+// optional filters user_id, client_id, kind, from, to (RFC3339 or epoch
+// millis) plus limit/offset. Newest-first; the SPA reverses for a
+// chronological timeline + export.
+func (h *handlers) handleListAdminDiagnostics(w http.ResponseWriter, r *http.Request) {
+	q := r.URL.Query()
+	params, ok := h.buildDiagnosticsFilter(w, q)
+	if !ok {
+		return
+	}
+	rows, err := dbq.New(h.pool).ListAdminDiagnostics(r.Context(), params)
+	if err != nil {
+		writeErrWithLog(w, h.logger, "admin: list diagnostics failed", apierror.Internal(err))
+		return
+	}
+	out := make([]adminDiagnosticView, 0, len(rows))
+	for _, row := range rows {
+		out = append(out, adminDiagnosticView{
+			ID:         uuidToString(row.ID),
+			UserID:     uuidToString(row.UserID),
+			Username:   row.Username,
+			ClientID:   row.ClientID,
+			AppVersion: row.AppVersion,
+			OsVersion:  row.OsVersion,
+			Kind:       row.Kind,
+			Payload:    json.RawMessage(row.Payload),
+			OccurredAt: formatTimestamp(row.OccurredAt),
+			ReceivedAt: formatTimestamp(row.ReceivedAt),
+		})
+	}
+	writeJSON(w, http.StatusOK, out)
+}
+
+// buildDiagnosticsFilter parses the query string into a query param
+// struct, writing a 400 and returning ok=false on a malformed time.
+func (h *handlers) buildDiagnosticsFilter(
+	w http.ResponseWriter, q map[string][]string,
+) (dbq.ListAdminDiagnosticsParams, bool) {
+	get := func(k string) string {
+		if v, present := q[k]; present && len(v) > 0 {
+			return v[0]
+		}
+		return ""
+	}
+	limit, offset, err := parseDiagPaging(get("limit"), get("offset"))
+	if err != nil {
+		writeErr(w, apierror.BadRequest("bad_request", "invalid limit or offset"))
+		return dbq.ListAdminDiagnosticsParams{}, false
+	}
+	from, okFrom := parseDiagTime(get("from"))
+	to, okTo := parseDiagTime(get("to"))
+	if !okFrom || !okTo {
+		writeErr(w, apierror.BadRequest("bad_request", "invalid from/to timestamp"))
+		return dbq.ListAdminDiagnosticsParams{}, false
+	}
+	params := dbq.ListAdminDiagnosticsParams{
+		ClientID: optionalQuery(get("client_id")),
+		Kind:     optionalQuery(get("kind")),
+		FromTs:   from,
+		ToTs:     to,
+		Off:      int32(offset),
+		Lim:      int32(limit),
+	}
+	if uid, valid := parseUUID(get("user_id")); valid {
+		params.UserID = uid
+	}
+	return params, true
+}
+
+// adminDiagnosticDeviceView is one device in the "who is reporting"
+// overview / device filter dropdown.
+type adminDiagnosticDeviceView struct {
+	ClientID   string `json:"client_id"`
+	UserID     string `json:"user_id"`
+	Username   string `json:"username"`
+	AppVersion string `json:"app_version"`
+	OsVersion  string `json:"os_version"`
+	LastSeen   string `json:"last_seen"`
+	EventCount int64  `json:"event_count"`
+}
+
+// handleListAdminDiagnosticDevices implements GET
+// /api/admin/diagnostics/devices?user_id=. Lists distinct reporting
+// devices so the admin UI can populate its device filter.
+func (h *handlers) handleListAdminDiagnosticDevices(w http.ResponseWriter, r *http.Request) {
+	var userID pgtype.UUID
+	if uid, valid := parseUUID(r.URL.Query().Get("user_id")); valid {
+		userID = uid
+	}
+	rows, err := dbq.New(h.pool).ListDiagnosticDevices(r.Context(), userID)
+	if err != nil {
+		writeErrWithLog(w, h.logger, "admin: list diagnostic devices failed", apierror.Internal(err))
+		return
+	}
+	out := make([]adminDiagnosticDeviceView, 0, len(rows))
+	for _, row := range rows {
+		out = append(out, adminDiagnosticDeviceView{
+			ClientID:   row.ClientID,
+			UserID:     uuidToString(row.UserID),
+			Username:   row.Username,
+			AppVersion: row.AppVersion,
+			OsVersion:  row.OsVersion,
+			LastSeen:   formatTimestamp(row.LastSeen),
+			EventCount: row.EventCount,
+		})
+	}
+	writeJSON(w, http.StatusOK, out)
+}
+
+// adminDebugModeReq is the body of PUT /api/admin/users/{id}/debug-mode.
+type adminDebugModeReq struct {
+	Enabled bool `json:"enabled"`
+}
+
+// handleAdminDebugModeToggle implements PUT
+// /api/admin/users/{id}/debug-mode — flip an account's diagnostics
+// opt-in remotely while a bug is live. Echoes the updated user.
+func (h *handlers) handleAdminDebugModeToggle(w http.ResponseWriter, r *http.Request) {
+	targetID, ok := requireURLUUID(w, r, "id")
+	if !ok {
+		return
+	}
+	var req adminDebugModeReq
+	if !decodeBody(w, r, &req) {
+		return
+	}
+	updated, err := dbq.New(h.pool).SetDebugMode(r.Context(), dbq.SetDebugModeParams{
+		ID:               targetID,
+		DebugModeEnabled: req.Enabled,
+	})
+	if err != nil {
+		writeErrWithLog(w, h.logger, "admin: set debug mode failed", apierror.Internal(err))
+		return
+	}
+	h.logger.Info("admin: debug mode toggled",
+		"target_user", uuidToString(targetID), "enabled", req.Enabled)
+	writeJSON(w, http.StatusOK, adminUserView{
+		ID:                  uuidToString(updated.ID),
+		Username:            updated.Username,
+		DisplayName:         updated.DisplayName,
+		IsAdmin:             updated.IsAdmin,
+		AutoApproveRequests: updated.AutoApproveRequests,
+		DebugModeEnabled:    updated.DebugModeEnabled,
+		CreatedAt:           updated.CreatedAt.Time.UTC().Format(time.RFC3339),
+	})
+}
+
+// parseDiagPaging reads limit/offset with diagnostics-specific caps.
+// Blank → defaults; out-of-range clamps; non-numeric → error.
+func parseDiagPaging(rawLimit, rawOffset string) (limit, offset int, err error) {
+	limit = defaultDiagnosticPageSize
+	if s := strings.TrimSpace(rawLimit); s != "" {
+		n, perr := strconv.Atoi(s)
+		if perr != nil {
+			return 0, 0, perr
+		}
+		if n < 1 {
+			n = 1
+		}
+		if n > maxDiagnosticPageSize {
+			n = maxDiagnosticPageSize
+		}
+		limit = n
+	}
+	if s := strings.TrimSpace(rawOffset); s != "" {
+		n, perr := strconv.Atoi(s)
+		if perr != nil {
+			return 0, 0, perr
+		}
+		if n > 0 {
+			offset = n
+		}
+	}
+	return limit, offset, nil
+}
+
+// optionalQuery returns nil for an absent/blank query value so the sqlc
+// NULL-or-equals filter treats it as "no filter."
+func optionalQuery(raw string) *string {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return nil
+	}
+	return &raw
+}
+
+// parseDiagTime accepts RFC3339 or epoch-millis. Empty = absent (NULL
+// filter, ok=true). Returns ok=false only on a non-empty unparseable value.
+func parseDiagTime(raw string) (pgtype.Timestamptz, bool) {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return pgtype.Timestamptz{}, true
+	}
+	if t, err := time.Parse(time.RFC3339, raw); err == nil {
+		return pgtype.Timestamptz{Time: t.UTC(), Valid: true}, true
+	}
+	if ms, err := strconv.ParseInt(raw, 10, 64); err == nil {
+		return pgtype.Timestamptz{Time: time.UnixMilli(ms).UTC(), Valid: true}, true
+	}
+	return pgtype.Timestamptz{}, false
+}

internal/api/me_profile.go

@@ -35,6 +35,10 @@ type meProfileResp struct {
 	DisplayName *string `json:"display_name"`
 	Email       *string `json:"email"`
 	IsAdmin     bool    `json:"is_admin"`
+	// DebugModeEnabled is the account's diagnostics opt-in. The client
+	// reads it here to decide whether to run the on-device diagnostics
+	// reporter (M9). Admin-set; the client also has a local OFF switch.
+	DebugModeEnabled bool `json:"debug_mode_enabled"`
 }
 
 func (h *handlers) handleUpdateMyProfile(w http.ResponseWriter, r *http.Request) {
@@ -120,10 +124,11 @@ func (h *handlers) handleUpdateMyProfile(w http.ResponseWriter, r *http.Request)
 
 func profileViewFromUser(u dbq.User) meProfileResp {
 	return meProfileResp{
-		ID:          uuidToString(u.ID),
-		Username:    u.Username,
-		DisplayName: u.DisplayName,
-		Email:       u.Email,
-		IsAdmin:     u.IsAdmin,
+		ID:               uuidToString(u.ID),
+		Username:         u.Username,
+		DisplayName:      u.DisplayName,
+		Email:            u.Email,
+		IsAdmin:          u.IsAdmin,
+		DebugModeEnabled: u.DebugModeEnabled,
 	}
 }