R7/analysis: RQ1/RQ2 confirmatory decision layer (frozen §6 gates in code)

Encodes the four lead-paper confirmatory tests exactly per the frozen prereg §6, calibrated on synthetic ground truth only (never re-fit to cell data): - RQ1-P1 leak: correlation AUC + BCa CI; gate = CI excludes 0.5 (D2); 0.60 CI lower bound = separate materiality label (material / weak-but-real), not the gate. - RQ1-P2 padding: paired ΔAUC = AUC(no-pad) − AUC(pad) over circuits; effective iff CI > 0. - RQ2-P1 anonymity set: ΔH = H(federated) − H(single, matched N) with Miller-Madow per-circuit entropy; two-sided grow / honest-shrink / inconclusive by CI sign. - RQ2-P3 mechanism: Spearman ρ(top-3 bridge concentration, per-circuit H) + CI. apply_holm corrects the reported RQ1/RQ2 subset against the full frozen family of 7 (family_size default 7) — never re-optimised to the 4 reported. Bootstrap p-values order the Holm step-down only; every decision is a CI gate, never a bare p. stats.py: bootstrap CIs can now return their resample distribution so the Holm ordering p-value comes from the same resamples as the CI. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-07-19 19:36:17 -07:00
parent 06352ecf47
commit cb4b6869fc
3 changed files with 386 additions and 12 deletions
@@ -0,0 +1,149 @@
+"""RQ1/RQ2 confirmatory gates — validated on synthetic ground truth only.
+
+Each test constructs a distribution whose §6 verdict is known by construction
+(a clear leak / a null; padding that works / doesn't; federation that grows /
+shrinks / is flat; a funnelling mechanism) and asserts the frozen gate fires the
+right way. No confirmatory-cell data is involved. The Holm test pins that the
+reported RQ1/RQ2 subset is corrected against the full frozen family of 7.
+"""
+
+from cmd_chat.sor.analysis.confirm import (
+    FROZEN_FAMILY_SIZE,
+    PairedCircuit,
+    apply_holm,
+    rq1_p1_leak,
+    rq1_p2_padding,
+    rq2_p1_delta_h,
+    rq2_p3_funnel,
+)
+
+
+class _LCG:
+    def __init__(self, seed=0x2468ACE0):
+        self.s = seed
+    def u(self):
+        self.s = (1103515245 * self.s + 12345) & 0x7FFFFFFF
+        return self.s / 0x7FFFFFFF
+
+
+def _pairs(hi_mean, lo_mean, n=80, rng=None):
+    """n linked pairs near hi_mean + n unlinked near lo_mean, with jitter."""
+    rng = rng or _LCG()
+    out = []
+    for _ in range(n):
+        out.append((hi_mean + 0.1 * rng.u(), True))
+        out.append((lo_mean + 0.1 * rng.u(), False))
+    return out
+
+
+# --------------------------------------------------------------------------- #
+# RQ1-P1 — leak gate + materiality label.
+# --------------------------------------------------------------------------- #
+def test_rq1p1_material_leak():
+    t = rq1_p1_leak(_pairs(0.9, 0.1), seed=1, n_resamples=2000)
+    assert t.decision == "leak"
+    assert t.label == "material"       # CI lower bound >= 0.60
+    assert t.ci.excludes(0.5)
+
+
+def test_rq1p1_null_when_scores_overlap():
+    # Linked and unlinked drawn from the same band -> AUC ~ 0.5, CI spans it.
+    t = rq1_p1_leak(_pairs(0.5, 0.5), seed=1, n_resamples=2000)
+    assert t.decision == "null"
+    assert not t.ci.excludes(0.5)
+
+
+# --------------------------------------------------------------------------- #
+# RQ1-P2 — padding efficacy (paired ΔAUC).
+# --------------------------------------------------------------------------- #
+def test_rq1p2_padding_effective_when_pad_lowers_auc():
+    rng = _LCG(0x1111)
+    circuits = []
+    for _ in range(30):
+        nopad = tuple(_pairs(0.9, 0.1, n=4, rng=rng))   # strong linkage
+        pad = tuple(_pairs(0.5, 0.5, n=4, rng=rng))      # padding blurs it (AUC~0.5)
+        circuits.append(PairedCircuit(nopad, pad))
+    t = rq1_p2_padding(circuits, seed=2, n_resamples=2000)
+    assert t.decision == "padding-effective"
+    assert t.ci.strictly_greater(0.0)
+
+
+def test_rq1p2_padding_ineffective_when_no_change():
+    rng = _LCG(0x2222)
+    circuits = []
+    for _ in range(30):
+        nopad = tuple(_pairs(0.7, 0.3, n=4, rng=rng))
+        pad = tuple(_pairs(0.7, 0.3, n=4, rng=rng))      # identical regime
+        circuits.append(PairedCircuit(nopad, pad))
+    t = rq1_p2_padding(circuits, seed=2, n_resamples=2000)
+    assert t.decision == "padding-ineffective"
+    assert not t.ci.strictly_greater(0.0)
+
+
+# --------------------------------------------------------------------------- #
+# RQ2-P1 — ΔH two-sided (grow / shrink / inconclusive).
+# --------------------------------------------------------------------------- #
+def _uniform_circuits(n_circuits, n_senders, per=50):
+    return [[per] * n_senders for _ in range(n_circuits)]
+
+
+def _skewed_circuits(n_circuits, n_senders):
+    # One dominant sender -> low entropy.
+    return [[1000] + [1] * (n_senders - 1) for _ in range(n_circuits)]
+
+
+def test_rq2p1_grow_when_federation_is_more_uniform():
+    fed = _uniform_circuits(30, 8)         # high H (~3 bits)
+    single = _skewed_circuits(30, 8)        # low H
+    t = rq2_p1_delta_h(fed, single, seed=3, n_resamples=2000)
+    assert t.decision == "grow"
+    assert t.ci.strictly_greater(0.0)
+
+
+def test_rq2p1_honest_shrink_reported():
+    fed = _skewed_circuits(30, 8)           # federation funnels -> low H
+    single = _uniform_circuits(30, 8)       # matched-N single house, high H
+    t = rq2_p1_delta_h(fed, single, seed=3, n_resamples=2000)
+    assert t.decision == "shrink"
+    assert t.ci.strictly_less(0.0)
+
+
+def test_rq2p1_inconclusive_when_arms_match():
+    fed = _uniform_circuits(30, 8)
+    single = _uniform_circuits(30, 8)
+    t = rq2_p1_delta_h(fed, single, seed=3, n_resamples=2000)
+    assert t.decision == "inconclusive"
+    assert not t.ci.excludes(0.0)
+
+
+# --------------------------------------------------------------------------- #
+# RQ2-P3 — funnelling mechanism (Spearman).
+# --------------------------------------------------------------------------- #
+def test_rq2p3_funnel_negative_rho():
+    # Higher top-3 concentration -> lower per-circuit entropy.
+    conc = [i / 20.0 for i in range(20)]
+    h = [3.0 - c for c in conc]
+    t = rq2_p3_funnel(conc, h, seed=4, n_resamples=2000)
+    assert t.decision == "funnel"
+    assert t.ci.strictly_less(0.0)
+
+
+# --------------------------------------------------------------------------- #
+# Holm over the full frozen family (size 7) while reporting 4.
+# --------------------------------------------------------------------------- #
+def test_apply_holm_corrects_against_family_of_seven():
+    t1 = rq1_p1_leak(_pairs(0.95, 0.05), seed=1, n_resamples=1500)   # tiny p
+    t2 = rq1_p2_padding(
+        [PairedCircuit(tuple(_pairs(0.9, 0.1, n=4)), tuple(_pairs(0.5, 0.5, n=4)))
+         for _ in range(30)], seed=2, n_resamples=1500)
+    fed, single = _uniform_circuits(30, 8), _skewed_circuits(30, 8)
+    t3 = rq2_p1_delta_h(fed, single, seed=3, n_resamples=1500)
+    t4 = rq2_p3_funnel([i / 20.0 for i in range(20)],
+                       [3.0 - i / 20.0 for i in range(20)], seed=4, n_resamples=1500)
+    holm = apply_holm([t1, t2, t3, t4])
+    assert len(holm) == 4
+    # Smallest-p test gets the full-family multiplier of 7, not 4.
+    top = min(holm, key=lambda h: h.rank)
+    assert top.multiplier == FROZEN_FAMILY_SIZE == 7
+    mults = sorted(h.multiplier for h in holm)
+    assert mults == [4, 5, 6, 7]