minstrel/internal/api/metrics_significance_test.go

package api

import (
	"math"
	"testing"
)

func TestProportionDelta_ReproducesTheDiscoverCase(t *testing.T) {
	// 59 plays at 15.3% skip vs 70 at 28.6%: a 13.3pp gap that looks
	// confident and sits at p ≈ 0.06.
	d := proportionDelta(0.153, 59, 0.286, 70)
	if d == nil {
		t.Fatal("expected a delta for two real samples")
	}
	if math.Abs(d.DeltaPP-(-13.3)) > 0.1 {
		t.Errorf("DeltaPP = %.2f, want ≈ -13.3", d.DeltaPP)
	}
	// This is the assertion the whole task exists for: at these sample sizes
	// the margin swallows the difference.
	if d.Distinguishable {
		t.Errorf("13.3pp on n=59/70 reported as distinguishable (margin %.2f) — "+
			"this is exactly the false confidence #2495 set out to remove", d.MarginPP)
	}
	if d.MarginPP <= 13.3 {
		t.Errorf("MarginPP = %.2f, expected it to exceed the 13.3pp delta", d.MarginPP)
	}
}

// Same effect size, ~10x the volume: now it is real. Proves the flag tracks
// sample size rather than just the size of the gap.
func TestProportionDelta_SameGapBecomesDistinguishableWithVolume(t *testing.T) {
	d := proportionDelta(0.153, 600, 0.286, 700)
	if d == nil {
		t.Fatal("expected a delta")
	}
	if !d.Distinguishable {
		t.Errorf("13.3pp on n=600/700 should be distinguishable (margin %.2f)", d.MarginPP)
	}
}

func TestProportionDelta_SignAndDirection(t *testing.T) {
	// Surface skips MORE than baseline -> positive delta (worse for skip rate).
	worse := proportionDelta(0.40, 500, 0.25, 500)
	if worse == nil || worse.DeltaPP <= 0 {
		t.Fatalf("expected a positive delta, got %+v", worse)
	}
	better := proportionDelta(0.10, 500, 0.25, 500)
	if better == nil || better.DeltaPP >= 0 {
		t.Fatalf("expected a negative delta, got %+v", better)
	}
}

func TestProportionDelta_EmptySamples(t *testing.T) {
	if d := proportionDelta(0.2, 0, 0.3, 100); d != nil {
		t.Errorf("n1=0 produced a delta: %+v", d)
	}
	if d := proportionDelta(0.2, 100, 0.3, 0); d != nil {
		t.Errorf("n2=0 produced a delta: %+v", d)
	}
}

// Two degenerate rates have zero standard error, which would report a margin of
// 0 and therefore "distinguishable" for a delta of exactly 0. Reporting nothing
// is the honest answer.
func TestProportionDelta_DegenerateRates(t *testing.T) {
	if d := proportionDelta(0, 50, 0, 50); d != nil {
		t.Errorf("both rates 0 produced a delta: %+v", d)
	}
	if d := proportionDelta(1, 50, 1, 50); d != nil {
		t.Errorf("both rates 1 produced a delta: %+v", d)
	}
	// One degenerate side is still informative — the other side carries
	// variance.
	if d := proportionDelta(0, 200, 0.3, 200); d == nil {
		t.Error("one degenerate rate should still yield a delta")
	}
}

func TestMeanDelta(t *testing.T) {
	// Completion is bimodal, so ~0.16 variance (sd ≈ 0.4) is realistic.
	const v = 0.16
	thin := meanDelta(0.82, v, 59, 0.54, v, 70)
	if thin == nil {
		t.Fatal("expected a delta")
	}
	if math.Abs(thin.DeltaPP-28.0) > 0.1 {
		t.Errorf("DeltaPP = %.2f, want ≈ 28.0", thin.DeltaPP)
	}
	// 28pp is large enough to survive even a wide margin at this n.
	if !thin.Distinguishable {
		t.Errorf("28pp on n=59/70 with sd 0.4 should be distinguishable (margin %.2f)", thin.MarginPP)
	}
	// A small completion gap at the same volume should not be.
	small := meanDelta(0.56, v, 59, 0.54, v, 70)
	if small == nil {
		t.Fatal("expected a delta")
	}
	if small.Distinguishable {
		t.Errorf("2pp on n=59/70 reported as distinguishable (margin %.2f)", small.MarginPP)
	}
}

// A sample variance needs at least two observations per side.
func TestMeanDelta_NeedsTwoObservations(t *testing.T) {
	if d := meanDelta(0.8, 0.1, 1, 0.5, 0.1, 100); d != nil {
		t.Errorf("n1=1 produced a delta: %+v", d)
	}
	if d := meanDelta(0.8, 0.1, 100, 0.5, 0.1, 1); d != nil {
		t.Errorf("n2=1 produced a delta: %+v", d)
	}
}

func TestMeanDelta_ZeroVarianceBothSides(t *testing.T) {
	if d := meanDelta(0.8, 0, 50, 0.5, 0, 50); d != nil {
		t.Errorf("zero variance on both sides produced a delta: %+v", d)
	}
}

func TestSampleVariance(t *testing.T) {
	// Observations 0, 1: mean 0.5, sample variance 0.5.
	if got := sampleVariance(1.0, 1.0, 2); math.Abs(got-0.5) > 1e-9 {
		t.Errorf("sampleVariance = %v, want 0.5", got)
	}
	// Identical observations -> zero variance, and must not go negative through
	// floating-point cancellation.
	if got := sampleVariance(4.0, 4.0, 4); got != 0 {
		t.Errorf("identical observations gave variance %v, want 0", got)
	}
	if got := sampleVariance(0, 0, 1); got != 0 {
		t.Errorf("n=1 gave variance %v, want 0", got)
	}
}

// Clamping matters: a negative variance would become NaN in the square root and
// propagate into the JSON as a null-ish number.
func TestSampleVariance_NeverNegative(t *testing.T) {
	// sqSum slightly below sum²/n, as cancellation can produce.
	if got := sampleVariance(10.0, 24.999999999, 4); got < 0 {
		t.Errorf("variance went negative: %v", got)
	}
}

func TestNewDelta_BoundaryCountsAsDistinguishable(t *testing.T) {
	d := newDelta(5.0, 5.0)
	if !d.Distinguishable {
		t.Error("a delta exactly equal to its margin should count as distinguishable")
	}
	d = newDelta(4.999, 5.0)
	if d.Distinguishable {
		t.Error("a delta just inside its margin should not count as distinguishable")
	}
}