hack-house/tests/test_sor_confirm.py

"""RQ1/RQ2 confirmatory gates — validated on synthetic ground truth only.

Each test constructs a distribution whose §6 verdict is known by construction
(a clear leak / a null; padding that works / doesn't; federation that grows /
shrinks / is flat; a funnelling mechanism) and asserts the frozen gate fires the
right way. No confirmatory-cell data is involved. The Holm test pins that the
reported RQ1/RQ2 subset is corrected against the full frozen family of 7.
"""

from cmd_chat.sor.analysis.confirm import (
    FROZEN_FAMILY_SIZE,
    PairedCircuit,
    apply_holm,
    rq1_p1_leak,
    rq1_p2_padding,
    rq2_p1_delta_h,
    rq2_p3_funnel,
)


class _LCG:
    def __init__(self, seed=0x2468ACE0):
        self.s = seed
    def u(self):
        self.s = (1103515245 * self.s + 12345) & 0x7FFFFFFF
        return self.s / 0x7FFFFFFF


def _pairs(hi_mean, lo_mean, n=80, rng=None):
    """n linked pairs near hi_mean + n unlinked near lo_mean, with jitter."""
    rng = rng or _LCG()
    out = []
    for _ in range(n):
        out.append((hi_mean + 0.1 * rng.u(), True))
        out.append((lo_mean + 0.1 * rng.u(), False))
    return out


# --------------------------------------------------------------------------- #
# RQ1-P1 — leak gate + materiality label.
# --------------------------------------------------------------------------- #
def test_rq1p1_material_leak():
    t = rq1_p1_leak(_pairs(0.9, 0.1), seed=1, n_resamples=2000)
    assert t.decision == "leak"
    assert t.label == "material"       # CI lower bound >= 0.60
    assert t.ci.excludes(0.5)


def test_rq1p1_null_when_scores_overlap():
    # Linked and unlinked drawn from the same band -> AUC ~ 0.5, CI spans it.
    t = rq1_p1_leak(_pairs(0.5, 0.5), seed=1, n_resamples=2000)
    assert t.decision == "null"
    assert not t.ci.excludes(0.5)


# --------------------------------------------------------------------------- #
# RQ1-P2 — padding efficacy (paired ΔAUC).
# --------------------------------------------------------------------------- #
def test_rq1p2_padding_effective_when_pad_lowers_auc():
    rng = _LCG(0x1111)
    circuits = []
    for _ in range(30):
        nopad = tuple(_pairs(0.9, 0.1, n=4, rng=rng))   # strong linkage
        pad = tuple(_pairs(0.5, 0.5, n=4, rng=rng))      # padding blurs it (AUC~0.5)
        circuits.append(PairedCircuit(nopad, pad))
    t = rq1_p2_padding(circuits, seed=2, n_resamples=2000)
    assert t.decision == "padding-effective"
    assert t.ci.strictly_greater(0.0)


def test_rq1p2_padding_ineffective_when_no_change():
    rng = _LCG(0x2222)
    circuits = []
    for _ in range(30):
        nopad = tuple(_pairs(0.7, 0.3, n=4, rng=rng))
        pad = tuple(_pairs(0.7, 0.3, n=4, rng=rng))      # identical regime
        circuits.append(PairedCircuit(nopad, pad))
    t = rq1_p2_padding(circuits, seed=2, n_resamples=2000)
    assert t.decision == "padding-ineffective"
    assert not t.ci.strictly_greater(0.0)


# --------------------------------------------------------------------------- #
# RQ2-P1 — ΔH two-sided (grow / shrink / inconclusive).
# --------------------------------------------------------------------------- #
def _uniform_circuits(n_circuits, n_senders, per=50):
    return [[per] * n_senders for _ in range(n_circuits)]


def _skewed_circuits(n_circuits, n_senders):
    # One dominant sender -> low entropy.
    return [[1000] + [1] * (n_senders - 1) for _ in range(n_circuits)]


def test_rq2p1_grow_when_federation_is_more_uniform():
    fed = _uniform_circuits(30, 8)         # high H (~3 bits)
    single = _skewed_circuits(30, 8)        # low H
    t = rq2_p1_delta_h(fed, single, seed=3, n_resamples=2000)
    assert t.decision == "grow"
    assert t.ci.strictly_greater(0.0)


def test_rq2p1_honest_shrink_reported():
    fed = _skewed_circuits(30, 8)           # federation funnels -> low H
    single = _uniform_circuits(30, 8)       # matched-N single house, high H
    t = rq2_p1_delta_h(fed, single, seed=3, n_resamples=2000)
    assert t.decision == "shrink"
    assert t.ci.strictly_less(0.0)


def test_rq2p1_inconclusive_when_arms_match():
    fed = _uniform_circuits(30, 8)
    single = _uniform_circuits(30, 8)
    t = rq2_p1_delta_h(fed, single, seed=3, n_resamples=2000)
    assert t.decision == "inconclusive"
    assert not t.ci.excludes(0.0)


# --------------------------------------------------------------------------- #
# RQ2-P3 — funnelling mechanism (Spearman).
# --------------------------------------------------------------------------- #
def test_rq2p3_funnel_negative_rho():
    # Higher top-3 concentration -> lower per-circuit entropy.
    conc = [i / 20.0 for i in range(20)]
    h = [3.0 - c for c in conc]
    t = rq2_p3_funnel(conc, h, seed=4, n_resamples=2000)
    assert t.decision == "funnel"
    assert t.ci.strictly_less(0.0)


# --------------------------------------------------------------------------- #
# Holm over the full frozen family (size 7) while reporting 4.
# --------------------------------------------------------------------------- #
def test_apply_holm_corrects_against_family_of_seven():
    t1 = rq1_p1_leak(_pairs(0.95, 0.05), seed=1, n_resamples=1500)   # tiny p
    t2 = rq1_p2_padding(
        [PairedCircuit(tuple(_pairs(0.9, 0.1, n=4)), tuple(_pairs(0.5, 0.5, n=4)))
         for _ in range(30)], seed=2, n_resamples=1500)
    fed, single = _uniform_circuits(30, 8), _skewed_circuits(30, 8)
    t3 = rq2_p1_delta_h(fed, single, seed=3, n_resamples=1500)
    t4 = rq2_p3_funnel([i / 20.0 for i in range(20)],
                       [3.0 - i / 20.0 for i in range(20)], seed=4, n_resamples=1500)
    holm = apply_holm([t1, t2, t3, t4])
    assert len(holm) == 4
    # Smallest-p test gets the full-family multiplier of 7, not 4.
    top = min(holm, key=lambda h: h.rank)
    assert top.multiplier == FROZEN_FAMILY_SIZE == 7
    mults = sorted(h.multiplier for h in holm)
    assert mults == [4, 5, 6, 7]