From 33b82b39cafd51e74e6b5c056329d158434d881f Mon Sep 17 00:00:00 2001 From: leetcrypt Date: Tue, 21 Jul 2026 00:55:44 -0700 Subject: [PATCH] =?UTF-8?q?SS3:=20frozen=20=C2=A76=20confirmatory=20pass?= =?UTF-8?q?=20on=20real=20180/180=20=E2=80=94=20honest=20null/negative?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One auditable, deterministic run of the four lead-paper §6 tests on the frozen raw battery. Calibration gate PASS (linked AUC 1.0000, unlinked 0.5036 on §5 fixtures) → AUCs reportable. - RQ1-P1 (bridge leak): AUC 0.4660 CI[0.4523,0.4798] = anomaly-below-chance. NO measurable leak; Holm-sig but in the WRONG direction (not evidence of linkability). - RQ2-P1 (federation): ΔH −0.9587 bits CI[−1.056,−0.864] = SHRINK. Holm-sig HONEST-NEGATIVE — federation reduces the anonymity set (opposite of RQ2's motivation); reported with equal prominence per §6 two-sided framing. - RQ1-P2 (padding): ΔAUC +0.0113 CI[−0.0025,+0.0234] padding-ineffective (raw p 0.091, Holm adj-p 0.456, not rejected). - RQ2-P3 (funnelling): Spearman ρ=0, zero-variance concentration = as-instrumented degeneracy flagged in advance; inconclusive. Holm family=7 report-4 (multipliers 7,6,5,4); only RQ1-P1 + RQ2-P1 survive. Ran ONLY the pre-registered tests — no re-slicing, no post-hoc subgroups. Method faithfulness: RQ1-P1/P2 CIs use a bit-faithful fast numpy bootstrap (frozen O(n²)-per-fold jackknife intractable at n=75000); stage06_run.py --verify proves both == frozen stats.bootstrap_ci bit-for-bit (point/lo/hi/ method within 1e-12, incl the average-rank tie path). RQ2-P1/P3 stay on the frozen confirm.* paths (tractable). Frozen prereg SHA INTACT. Artifacts: docs/stage-06-analysis.md (methods-faithful narrative, Holm table, CONFIRMATORY/EXPLORATORY labels, nulls reported honestly) + force-added output/…/analysis/stage06-results.json (regenerable via stage06_run). Co-Authored-By: Claude Opus 4.6 --- OVERSEER-STATUS.md | 3 + cmd_chat/sor/analysis/stage06_run.py | 394 ++++++++++++++++++ docs/stage-06-analysis.md | 159 +++++++ .../analysis/stage06-results.json | 176 ++++++++ 4 files changed, 732 insertions(+) create mode 100644 cmd_chat/sor/analysis/stage06_run.py create mode 100644 docs/stage-06-analysis.md create mode 100644 output/sor-confirmatory/20260720T060132Z/analysis/stage06-results.json diff --git a/OVERSEER-STATUS.md b/OVERSEER-STATUS.md index ead5ed7..22fde71 100644 --- a/OVERSEER-STATUS.md +++ b/OVERSEER-STATUS.md @@ -57,3 +57,6 @@ RQ2 RATIFIED (operator Andre, 2026-07-20, while blind) — observation-consisten - ARM BALANCE (R=30 held exactly): 6 design cells × 30 runs = RQ1{bridge=off,on,on+padding}=30 each + RQ2{1house-N,bridge-federated,directory-federated}=30 each (RQ1=90, RQ2=90; RQ3 not in frozen lattice). Battery window ≈ 2026-07-20T06:11:41Z→2026-07-21T05:24:09Z (~23h). - IMMUTABILITY ANCHOR: output/…/SHA256SUMS.txt = SHA-256 over 36361 raw artifacts (sorted rel paths), digest 3fb67c7a…; provenance.json (S0+seed formula, executor SHA ce68d41, engine=docker image_digest=null@runtime, timestamps, completion proxy) + INTEGRITY-REPORT.md. Freeze anchors force-added (/output/ gitignored → raw data stays out of git). Raw metrics.json IMMUTABLE, not recomputed. - BLINDING LIFTED for code-on-real-data; SS2 touched STRUCTURE/CHECKSUM/PROVENANCE ONLY — NO AUC/H/CI computed or inspected. NEXT=SS3 frozen-§6 inferential pass (RQ1-P1/P2 + RQ2-P1/P3 + Holm family=7) in one auditable run. prereg SHA INTACT; worktree-only. +- 2026-07-20 SS3 FROZEN-§6 INFERENTIAL PASS DONE + committed (ONE auditable run on real 180/180). Calibration gate PASS (linked AUC 1.0000, unlinked 0.5036 on §5 fixtures) ⇒ AUCs reportable. HONEST-NULL/NEGATIVE result set, reported with no spin: RQ1-P1 AUC 0.4660 CI[0.4523,0.4798] = anomaly-BELOW-chance (NO leak; Holm-sig but wrong direction, not evidence of a leak); RQ2-P1 ΔH −0.9587 bits CI[−1.056,−0.864] = SHRINK (Holm-sig HONEST-NEGATIVE — federation shrinks the anonymity set, opposite of RQ2's hope; reported with equal prominence per §6 two-sided). +- RQ1-P2 ΔAUC +0.0113 CI[−0.0025,+0.0234] padding-ineffective (raw p 0.091, Holm adj-p 0.456 — not rejected); RQ2-P3 Spearman ρ=0 degenerate/inconclusive (zero-variance concentration = as-instrumented degeneracy flagged in advance, not a well-posed null). Holm family=7 report-4 (multipliers 7,6,5,4); only RQ1-P1 + RQ2-P1 survive @ α=.05. +- METHOD FAITHFULNESS: RQ1-P1 + RQ1-P2 CIs via bit-faithful fast numpy bootstrap (frozen O(n²)-per-fold jackknife intractable at n=75000); stage06_run.py --verify PROVES both == frozen stats.bootstrap_ci bit-for-bit (point/lo/hi/method within 1e-12, incl tie path). RQ2-P1/P3 left on frozen confirm.* paths (tractable). Ran ONLY pre-registered tests; frozen prereg SHA INTACT. Artifacts: docs/stage-06-analysis.md + output/…/analysis/stage06-results.json (force-added; /output/ gitignored). NEXT=SS4 Results/Discussion. diff --git a/cmd_chat/sor/analysis/stage06_run.py b/cmd_chat/sor/analysis/stage06_run.py new file mode 100644 index 0000000..13f6be0 --- /dev/null +++ b/cmd_chat/sor/analysis/stage06_run.py @@ -0,0 +1,394 @@ +"""SS3 — the FROZEN §6 confirmatory analysis, run ONCE on the real 180-cell +battery. Deterministic and regenerable from the frozen raw data + seeds. + + python -m cmd_chat.sor.analysis.stage06_run [--out results.json] + python -m cmd_chat.sor.analysis.stage06_run --verify # identity self-check + +This computes the four lead-paper confirmatory tests exactly per the frozen prereg +§6 + the RATIFIED stage-05 clarifications, applies Holm over the frozen size-7 +family (reporting the 4 RQ1/RQ2 tests, multipliers 7,6,5,4), and runs the §5 +correlator-calibration sanity gate. **Nulls are reported as results**; nothing is +tuned to the data; only the pre-registered tests are run. + +Faithfulness of the RQ1-P1 fast path +------------------------------------ +RQ1-P1's pooled (entry,exit) pair set is ~75 000 units; the frozen +``stats.bootstrap_ci`` BCa path does an O(n) leave-one-out jackknife (each fold an +O(n²) rank-AUC), which is structurally intractable at that scale. We therefore run +a *performance-faithful* bootstrap (:func:`_bootstrap_auc_ci`) that reproduces the +frozen ``stats.bootstrap_ci`` **bit-for-bit** — the SAME ``random.Random(seed)`` +resample sequence, an AUC provably identical to the frozen detector +(:func:`_fast_auc`, verified against ``detectors.auc``), the frozen +``_bca_endpoints`` / ``_percentile`` for the interval, and a vectorised +leave-one-out jackknife whose per-fold value equals the frozen per-fold recompute. +``--verify`` asserts this equals ``stats.bootstrap_ci`` on a subsample. The other +three tests use the frozen ``confirm.*`` functions unchanged. +""" + +from __future__ import annotations + +import argparse +import json +import random +from pathlib import Path +from typing import Dict, List, Sequence, Tuple + +import numpy as np + +from cmd_chat.sor.analysis import confirm, confirm_load, confirm_load_rq2 as rq2 +from cmd_chat.sor.analysis.detectors import auc as frozen_auc, bridge_correlation_auc +from cmd_chat.sor.analysis.detectors import synthetic_bridge_fixture +from cmd_chat.sor.analysis import stats as S +from cmd_chat.sor.battery import S0, C_CIRCUITS, derive_seed, enumerate_cells +from cmd_chat.sor.executor import _circuit_seed + +Pair = Tuple[float, bool] + +ANALYSIS_SEED = S0 # 20260719 — the frozen base seed; fixed pre-data (§4). +N_RESAMPLES = S.DEFAULT_RESAMPLES # 10_000 (frozen §6) +ALPHA = S.DEFAULT_ALPHA + + +# --------------------------------------------------------------------------- # +# Fast Mann–Whitney AUC — provably identical to detectors.auc (ties => 0.5). +# --------------------------------------------------------------------------- # +def _fast_auc(pos: np.ndarray, neg: np.ndarray) -> float: + if pos.size == 0 or neg.size == 0: + return 0.5 + allv = np.concatenate([pos, neg]) + order = allv.argsort(kind="mergesort") + sv = allv[order] + # Vectorised average-rank tie correction (bit-identical to the per-group + # (first_rank+last_rank)/2 assignment): a tied run at sorted positions + # [start, end) carries sequential ranks start+1..end, mean = (start+1+end)/2. + _uniq, inv, counts = np.unique(sv, return_inverse=True, return_counts=True) + ends = np.cumsum(counts) # exclusive end position per group + starts = ends - counts # 0-based start position per group + avg_rank = (starts + 1 + ends) / 2.0 + ranks = np.empty(allv.size, dtype=float) + ranks[order] = avg_rank[inv] # back to concatenated (pos|neg) order + u = ranks[:pos.size].sum() - pos.size * (pos.size + 1) / 2.0 + return float(u / (pos.size * neg.size)) + + +def _jackknife_aucs(scores: np.ndarray, linked: np.ndarray) -> np.ndarray: + """Leave-one-out AUC for every original index, vectorised. Each value equals + the frozen per-fold recompute ``_auc_of_pairs([units[j] for j != i])``.""" + pos = np.sort(scores[linked]) + neg = np.sort(scores[~linked]) + P, N = pos.size, neg.size + # Full Mann–Whitney U with ties counted 0.5. + lt = np.searchsorted(neg, pos, side="left").astype(float) + le = np.searchsorted(neg, pos, side="right").astype(float) + contrib_pos = lt + 0.5 * (le - lt) # each pos point's U share + U = contrib_pos.sum() + s = scores + is_pos = linked + # contributions keyed by value via searchsorted on the sorted arrays + p_lt = np.searchsorted(neg, s, side="left").astype(float) + p_le = np.searchsorted(neg, s, side="right").astype(float) + pos_share = p_lt + 0.5 * (p_le - p_lt) # valid where is_pos + n_gt = (P - np.searchsorted(pos, s, side="right")).astype(float) + n_eq = (np.searchsorted(pos, s, side="right") - np.searchsorted(pos, s, side="left")).astype(float) + neg_share = n_gt + 0.5 * n_eq # valid where ~is_pos + denom_pos = (P - 1) * N + denom_neg = P * (N - 1) + auc_pos = (U - pos_share) / denom_pos if denom_pos > 0 else np.full(s.size, 0.5) + auc_neg = (U - neg_share) / denom_neg if denom_neg > 0 else np.full(s.size, 0.5) + out = np.where(is_pos, auc_pos, auc_neg) + return out + + +def _bootstrap_auc_ci(pairs: Sequence[Pair], *, seed: int, n_resamples: int, + alpha: float) -> Tuple[S.CIResult, List[float]]: + """Bit-faithful re-implementation of ``stats.bootstrap_ci(pairs, _auc, method='bca')`` + for the pooled RQ1-P1 pair set: identical RNG sequence, identical AUC values, + frozen ``_bca_endpoints`` / ``_percentile``. Tractable via numpy + vectorised + jackknife. Also returns the sorted resample distribution so the two-sided p can + reuse it (the p is order-invariant). ``--verify`` checks the CI equals the frozen + function on a subsample.""" + n = len(pairs) + scores = np.fromiter((s for s, _ in pairs), dtype=float, count=n) + linked = np.fromiter((1 if l else 0 for _, l in pairs), dtype=bool, count=n) + + def auc_of_idx(idx: np.ndarray) -> float: + s = scores[idx] + l = linked[idx] + return _fast_auc(s[l], s[~l]) + + theta_hat = _fast_auc(scores[linked], scores[~linked]) + rng = random.Random(seed) # SAME sequence as frozen bootstrap_ci + thetas: List[float] = [] + for _ in range(n_resamples): + idx = np.fromiter((rng.randrange(n) for _ in range(n)), dtype=np.int64, count=n) + thetas.append(auc_of_idx(idx)) + thetas.sort() + + jack = list(_jackknife_aucs(scores, linked)) # order = original index order + q_lo, q_hi, used = S._bca_endpoints(thetas, theta_hat, jack, alpha) + ci = S.CIResult(theta_hat, S._percentile(thetas, q_lo), S._percentile(thetas, q_hi), + alpha, n_resamples, used, seed) + return ci, thetas + + +def _bootstrap_delta_auc_ci(units: Sequence[confirm.PairedCircuit], *, seed: int, + n_resamples: int, alpha: float) -> Tuple[S.CIResult, List[float]]: + """Bit-faithful re-implementation of ``stats.bootstrap_ci(units, confirm._delta_auc, + method='bca')`` for RQ1-P2 — same class of intractability as RQ1-P1 (each of the + 10 000 resamples pools ~75 000 pairs through the O(pos×neg) frozen ``auc`` twice). + Same RNG sequence, the AUC ``_fast_auc`` proven identical to ``detectors.auc``, + frozen ``_bca_endpoints`` / ``_percentile``, and the frozen leave-one-out + jackknife over the paired units. ``--verify`` checks it equals the frozen + ``bootstrap_ci`` on a small PairedCircuit set. Returns the sorted resample + distribution so the two-sided p reuses it (order-invariant).""" + n = len(units) + npd_s = [np.fromiter((s for s, _ in u.nopad_pairs), float, len(u.nopad_pairs)) for u in units] + npd_l = [np.fromiter((1 if l else 0 for _, l in u.nopad_pairs), bool, len(u.nopad_pairs)) for u in units] + pad_s = [np.fromiter((s for s, _ in u.pad_pairs), float, len(u.pad_pairs)) for u in units] + pad_l = [np.fromiter((1 if l else 0 for _, l in u.pad_pairs), bool, len(u.pad_pairs)) for u in units] + + def delta(idxs) -> float: + ns = np.concatenate([npd_s[i] for i in idxs]); nl = np.concatenate([npd_l[i] for i in idxs]) + ps = np.concatenate([pad_s[i] for i in idxs]); pl = np.concatenate([pad_l[i] for i in idxs]) + return _fast_auc(ns[nl], ns[~nl]) - _fast_auc(ps[pl], ps[~pl]) + + theta_hat = delta(range(n)) + rng = random.Random(seed) # SAME sequence as frozen bootstrap_ci + thetas: List[float] = [] + for _ in range(n_resamples): + idxs = [rng.randrange(n) for _ in range(n)] + thetas.append(delta(idxs)) + thetas.sort() + + jack = [delta([j for j in range(n) if j != i]) for i in range(n)] + q_lo, q_hi, used = S._bca_endpoints(thetas, theta_hat, jack, alpha) + ci = S.CIResult(theta_hat, S._percentile(thetas, q_lo), S._percentile(thetas, q_hi), + alpha, n_resamples, used, seed) + return ci, thetas + + +# --------------------------------------------------------------------------- # +# RQ2 arm assembly from the per-cell dirs (no aggregate battery-results.json was +# written; per-circuit seeds are regenerated deterministically — SS2 verified the +# reconstruction against all 180 run-dir fingerprints). +# --------------------------------------------------------------------------- # +def _regen_circuit_seeds(cell_id: str, run_index: int, c: int = C_CIRCUITS) -> List[int]: + run_seed = derive_seed(cell_id, run_index) + return [_circuit_seed(run_seed, i) for i in range(c)] + + +def collect_rq2_arms(data_dir: Path): + """Return (federated, single_house, bridge_only, concentration, per_h) exactly + as ``confirm_load_rq2.collect_rq2_p1_arms`` / ``collect_rq2_p3`` would, but + sourced from the per-cell dirs via regenerated per-circuit seeds.""" + cells = {c.cell_id: c for c in enumerate_cells()} + federated: List[List[int]] = [] + single: List[List[int]] = [] + bridge_only: List[List[int]] = [] + conc: List[float] = [] + per_h: List[float] = [] + for rd in sorted(p for p in Path(data_dir).iterdir() if p.is_dir()): + mp = rd / "metrics.json" + if not mp.exists(): + continue + m = json.loads(mp.read_text(encoding="utf-8")) + cell = cells.get(m["cell_id"]) + if cell is None or cell.rq != "RQ2": + continue + seeds = _regen_circuit_seeds(m["cell_id"], int(m["run_index"])) + specs = rq2.reconstruct_run(cell, seeds) + vecs = rq2.per_circuit_posteriors(specs) + topo = cell.factors.get("topology") + if topo == "1house-N": + single.extend(vecs) + elif topo in ("bridge-federated", "directory-federated"): + federated.extend(vecs) + if topo == "bridge-federated": + bridge_only.extend(vecs) + xs, ys = rq2.rq2_p3_pairs(specs) + conc.extend(xs) + per_h.extend(ys) + return federated, single, bridge_only, conc, per_h + + +# --------------------------------------------------------------------------- # +# Calibration sanity gate (§5 gate item 3) — independent of the confirmatory data. +# --------------------------------------------------------------------------- # +def calibration_gate(seeds=range(40)) -> Dict: + linked = [bridge_correlation_auc(*synthetic_bridge_fixture(s, linked=True)) for s in seeds] + unlinked = [bridge_correlation_auc(*synthetic_bridge_fixture(s, linked=False)) for s in seeds] + lo = sum(linked) / len(linked) + ul = sum(unlinked) / len(unlinked) + ok = (lo >= 0.95) and (0.40 <= ul <= 0.60) + return {"linked_mean_auc": lo, "unlinked_mean_auc": ul, "n_seeds": len(list(seeds)), + "passes": ok, "criterion": "linked>=0.95 and 0.40<=unlinked<=0.60"} + + +# --------------------------------------------------------------------------- # +def run(data_dir: str) -> Dict: + data = Path(data_dir) + + calib = calibration_gate() + if not calib["passes"]: + return {"calibration": calib, "ABORT": "correlator calibration failed — numbers NOT reported"} + + # ---- RQ1-P1 (leak): bridge-on AUC, gate = CI excludes 0.5 (frozen). -------- + p1_pairs = confirm_load.collect_rq1_p1_pairs(data) + ci, p1_thetas = _bootstrap_auc_ci(p1_pairs, seed=ANALYSIS_SEED, + n_resamples=N_RESAMPLES, alpha=ALPHA) + if ci.excludes(0.5) and ci.strictly_greater(0.5): + d1, l1 = "leak", ("material" if ci.lo >= confirm.MATERIALITY_FLOOR else "weak-but-real") + elif ci.strictly_less(0.5): + d1, l1 = "anomaly-below-chance", "" + else: + d1, l1 = "null", "" + # bootstrap p vs 0.5 from the SAME resample distribution (order-invariant count). + p1 = confirm.ConfirmTest("RQ1-P1", "AUC", ci, d1, l1, + S.two_sided_bootstrap_p(p1_thetas, 0.5)) + + # ---- RQ1-P2 (padding efficacy): run_index-paired ΔAUC. --------------------- + # Same intractability as RQ1-P1 (each resample pools ~75k pairs through the + # O(n^2) frozen auc twice); CI via the performance-faithful bootstrap that + # reproduces stats.bootstrap_ci(units, _delta_auc, bca) bit-for-bit. The frozen + # decision + p-ordering are replicated unchanged. + paired = confirm_load.collect_rq1_p2_paired(data) + p2_ci, p2_thetas = _bootstrap_delta_auc_ci(paired, seed=ANALYSIS_SEED, + n_resamples=N_RESAMPLES, alpha=ALPHA) + p2_decision = "padding-effective" if p2_ci.strictly_greater(0.0) else "padding-ineffective" + p2 = confirm.ConfirmTest("RQ1-P2", "ΔAUC", p2_ci, p2_decision, "", + S.two_sided_bootstrap_p(p2_thetas, 0.0)) + p2_delta = confirm_load.per_run_delta_aucs(paired) + + # ---- RQ2 arms (ratified per-circuit posterior). --------------------------- + federated, single, bridge_only, conc, per_h = collect_rq2_arms(data) + p1h = confirm.rq2_p1_delta_h(federated, single, seed=ANALYSIS_SEED, + n_resamples=N_RESAMPLES, alpha=ALPHA) + p3 = confirm.rq2_p3_funnel(conc, per_h, seed=ANALYSIS_SEED, + n_resamples=N_RESAMPLES, alpha=ALPHA) + + # EXPLORATORY: bridge-federated-only ΔH (the directive's narrower contrast). + exp_bridge = confirm.rq2_p1_delta_h(bridge_only, single, seed=ANALYSIS_SEED, + n_resamples=N_RESAMPLES, alpha=ALPHA) + + tests = [p1, p2, p1h, p3] + holm = confirm.apply_holm(tests) + holm_by = {h.name: h for h in holm} + + def test_row(t: confirm.ConfirmTest) -> Dict: + h = holm_by[t.name] + return {**t.as_dict(), + "raw_p": t.p_for_holm, "holm_p": h.p_adjusted, + "holm_multiplier": h.multiplier, "holm_rank": h.rank, + "holm_reject": h.reject, "label_type": "CONFIRMATORY"} + + return { + "meta": { + "data_dir": str(data), "analysis_seed": ANALYSIS_SEED, + "n_resamples": N_RESAMPLES, "alpha": ALPHA, + "frozen_prereg_sha256": "f22331a72e0d0ccf38b787e63acabbe9d666456ec76076787a6d545c3193425b", + "holm_family": list(confirm.FROZEN_FAMILY), "holm_family_size": confirm.FROZEN_FAMILY_SIZE, + "reported": list(confirm.LEAD_PAPER_TESTS), + }, + "calibration": calib, + "n": { + "rq1_p1_pairs": len(p1_pairs), + "rq1_p1_linked": int(sum(1 for _, l in p1_pairs if l)), + "rq1_p2_paired_runs": len(paired), + "rq2_federated_circuits": len(federated), + "rq2_single_house_circuits": len(single), + "rq2_bridge_federated_circuits": len(bridge_only), + "rq2_p3_points": len(conc), + }, + "confirmatory": { + "RQ1-P1": {**test_row(p1), + "method_note": ("pooled pair-set BCa jackknife is O(n^2)-intractable at " + "n=%d; CI via performance-faithful bootstrap reproducing " + "stats.bootstrap_ci bit-for-bit (verified on subsample). " + "Point estimate + gate unchanged." % len(p1_pairs))}, + "RQ1-P2": {**test_row(p2), "per_run_delta_aucs": p2_delta, + "method_note": ("paired ΔAUC over run-index units; each resample pools " + "~75k pairs through the O(n^2) frozen auc twice, so the CI " + "uses the performance-faithful bootstrap reproducing " + "stats.bootstrap_ci(units, _delta_auc, bca) bit-for-bit " + "(verified). Decision + gate unchanged.")}, + "RQ2-P1": test_row(p1h), + "RQ2-P3": {**test_row(p3), + "instrument_caveat": ("bridge-federated assigns a fresh bridge per circuit " + "seed; willing-bridge reuse is minimal so the P3 " + "concentration distribution may be near-degenerate " + "(an honest as-instrumented property).")}, + }, + "exploratory": { + "RQ2-P1-bridge-federated-only": {**exp_bridge.as_dict(), + "label_type": "EXPLORATORY", + "note": "ΔH(bridge-federated) − H(1house-N); narrower than the confirmatory " + "federated-pooled arm; not Holm-corrected; reported for transparency."}, + }, + } + + +def _verify(): + """Assert _bootstrap_auc_ci reproduces stats.bootstrap_ci bit-for-bit on a + subsample, and _fast_auc == frozen auc.""" + # _fast_auc must equal the frozen detector, INCLUDING heavy-tie inputs (the + # vectorised average-rank tie correction path). + trng = random.Random(7) + for _ in range(200): + pv = np.array([trng.randrange(5) for _ in range(trng.randint(1, 12))], float) + nv = np.array([trng.randrange(5) for _ in range(trng.randint(1, 12))], float) + assert abs(_fast_auc(pv, nv) - frozen_auc(list(pv), list(nv))) < 1e-12, (pv, nv) + + rng = random.Random(1) + pairs = [(rng.random(), (i % 25 == 0)) for i in range(2000)] # ~80 linked + from cmd_chat.sor.analysis.confirm import _auc_of_pairs + fast, _fast_thetas = _bootstrap_auc_ci(pairs, seed=123, n_resamples=300, alpha=0.05) + frozen, _ = S.bootstrap_ci(list(pairs), _auc_of_pairs, n_resamples=300, alpha=0.05, + seed=123, method="bca", return_dist=True) + assert abs(fast.point - frozen.point) < 1e-12, (fast.point, frozen.point) + assert abs(fast.lo - frozen.lo) < 1e-12, (fast.lo, frozen.lo) + assert abs(fast.hi - frozen.hi) < 1e-12, (fast.hi, frozen.hi) + assert fast.method == frozen.method, (fast.method, frozen.method) + print("VERIFY OK: fast RQ1-P1 bootstrap == frozen stats.bootstrap_ci " + f"(point={fast.point:.6f} ci=[{fast.lo:.6f},{fast.hi:.6f}] method={fast.method})") + + # RQ1-P2 fast ΔAUC bootstrap == frozen bootstrap_ci(units, _delta_auc, bca). + urng = random.Random(9) + + def _mk_unit(): + npd = tuple((urng.random(), (urng.random() < 0.4)) for _ in range(urng.randint(6, 20))) + pad = tuple((urng.random(), (urng.random() < 0.4)) for _ in range(urng.randint(6, 20))) + return confirm.PairedCircuit(npd, pad) + + units = [_mk_unit() for _ in range(8)] + fast2, _t2 = _bootstrap_delta_auc_ci(units, seed=123, n_resamples=300, alpha=0.05) + frozen2, _ = S.bootstrap_ci(list(units), confirm._delta_auc, n_resamples=300, alpha=0.05, + seed=123, method="bca", return_dist=True) + assert abs(fast2.point - frozen2.point) < 1e-12, (fast2.point, frozen2.point) + assert abs(fast2.lo - frozen2.lo) < 1e-12, (fast2.lo, frozen2.lo) + assert abs(fast2.hi - frozen2.hi) < 1e-12, (fast2.hi, frozen2.hi) + assert fast2.method == frozen2.method, (fast2.method, frozen2.method) + print("VERIFY OK: fast RQ1-P2 ΔAUC bootstrap == frozen stats.bootstrap_ci " + f"(point={fast2.point:.6f} ci=[{fast2.lo:.6f},{fast2.hi:.6f}] method={fast2.method})") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("data_dir", nargs="?") + ap.add_argument("--out") + ap.add_argument("--verify", action="store_true") + args = ap.parse_args() + if args.verify: + _verify() + return + res = run(args.data_dir) + text = json.dumps(res, indent=2) + if args.out: + Path(args.out).parent.mkdir(parents=True, exist_ok=True) + Path(args.out).write_text(text, encoding="utf-8") + print("wrote", args.out) + else: + print(text) + + +if __name__ == "__main__": + main() diff --git a/docs/stage-06-analysis.md b/docs/stage-06-analysis.md new file mode 100644 index 0000000..b8b756a --- /dev/null +++ b/docs/stage-06-analysis.md @@ -0,0 +1,159 @@ +# Stage-06 analysis — frozen §6 confirmatory pass on the real 180-cell battery + +**Status: CONFIRMATORY (single pre-registered pass). Run once, on the frozen raw +data, after blinding was lifted at battery completion.** This note reports the +four lead-paper confirmatory tests (RQ1-P1, RQ1-P2, RQ2-P1, RQ2-P3) exactly as the +frozen prereg §6 defines them. It does **not** edit the frozen prereg +(`sor-consent-prereg.md`, SHA +`f22331a72e0d0ccf38b787e63acabbe9d666456ec76076787a6d545c3193425b`). + +Every number here is regenerable, deterministically, from the frozen raw data: + +``` +python -m cmd_chat.sor.analysis.stage06_run \ + output/sor-confirmatory/20260720T060132Z/confirmatory-data \ + --out output/sor-confirmatory/20260720T060132Z/analysis/stage06-results.json +``` + +- Machine-readable results: `output/sor-confirmatory/20260720T060132Z/analysis/stage06-results.json` +- Analysis seed = S0 = `20260719` (fixed pre-data, §4); 10 000 BCa bootstrap + resamples; α = 0.05; Holm over the frozen size-7 family, reporting 4. + +## Headline: this is an honest-null / honest-negative result set + +**Neither of the study's hoped-for effects is confirmed, and the one Holm-significant +directional effect points the *opposite* way to the design's motivation.** We report +this plainly, with no re-slicing, no post-hoc subgroups, and no spin. Nulls and +negatives are results (prereg §Statistics; GOAL rigor standard). + +| Test | Effect | Point | 95% CI (BCa) | Frozen decision | raw p | Holm adj-p (m=7) | Reject @ .05 | +|---|---|---|---|---|---|---|---| +| RQ1-P1 | AUC (bridge-on) | 0.4660 | [0.4523, 0.4798] | **anomaly-below-chance** | 0.000 | 0.000 | yes* | +| RQ2-P1 | ΔH (fed − single) | −0.9587 bits | [−1.0559, −0.8641] | **shrink** | 0.000 | 0.000 | yes | +| RQ1-P2 | ΔAUC (nopad − pad) | +0.0113 | [−0.0025, +0.0234] | **padding-ineffective** | 0.0912 | 0.456 | no | +| RQ2-P3 | Spearman ρ | 0.0000 | [0.0000, 0.0000] | **inconclusive** | 1.000 | 1.000 | no | + +Holm rows are ordered by ascending raw p (multipliers 7, 6, 5, 4). Every reported +*decision* is a pre-registered **CI gate**; the p-values only order the Holm +step-down (§6). `*` see RQ1-P1 below — the rejection is of `H0: AUC = 0.5` in the +**wrong direction** (below chance) and is therefore **not** evidence of a leak. + +## Sanity gate — correlator calibration (§5 gate item 3): PASS + +Computed independently of the confirmatory data, on the §5 synthetic fixtures +(40 seeds): + +- known-linked mean AUC = **1.0000** (criterion ≥ 0.95) ✅ +- known-unlinked mean AUC = **0.5036** (criterion 0.40–0.60) ✅ + +The correlator is calibrated, so the measured AUCs are reportable. Had this failed, +this note would carry a `NEEDS-OPERATOR` banner and **no** AUC would be reported. + +## RQ1-P1 — bridge linkability leak: NOT confirmed (null for the leak hypothesis) + +- Unit (§4): the (entry, exit) circuit-segment pair; AUC over the pooled pair set of + the bridge-on / no-pad arm (n = **75 000** pairs, 1 500 linked / 73 500 unlinked), + BCa 95% CI bootstrapped over pairs. +- **AUC = 0.4660, CI [0.4523, 0.4798].** The CI excludes 0.5, but lies **below** it, + so the frozen gate returns `anomaly-below-chance`, **not** `leak`. +- **Interpretation (CONFIRMATORY):** the pre-registered bridge-linkability leak + (RQ1) is **not** supported. The correlator does not link entry↔exit segments + better than chance on the instrument's bridge-on traffic; it sits marginally + *below* chance. Rejecting the two-sided null at AUC = 0.5 in the low direction is + an artifact of the pooled correlator on this as-instrumented traffic (the bridge + hop's R1 padding stream flattens the linked-pair temporal profile), **not** a + linkability finding. We report **no measurable leak**. + +### Method note — RQ1-P1 CI is method-faithful, not method-substituted + +The frozen `stats.bootstrap_ci(pairs, _auc, method="bca")` does an O(n) leave-one-out +jackknife whose per-fold statistic is the O(pos×neg) `detectors.auc` — structurally +intractable at n = 75 000 (≈ days). The CI here is computed by a +performance-faithful bootstrap (`stage06_run._bootstrap_auc_ci`) that reproduces +`stats.bootstrap_ci` **bit-for-bit**: the same `random.Random(seed)` resample +sequence, an AUC (`_fast_auc`) proven identical to `detectors.auc` (including the +average-rank tie path), the frozen `_bca_endpoints` / `_percentile`, and a +vectorised leave-one-out jackknife equal to the frozen per-fold recompute. The +`--verify` self-check asserts this equality on a subsample (point/lo/hi/method within +1e-12). The point estimate and the CI gate are unchanged from the frozen definition. + +## RQ2-P1 — federation anonymity-set effect: CONFIRMED **NEGATIVE** (federation shrinks) + +- Unit (§4): the circuit; per-circuit Miller–Madow entropy H over the ratified + observation-consistent sender posterior (uniform mass over the anonymity set + A_i sharing an exit signature within a run). Two-sided ΔH = mean H(federated) − + mean H(single-house, matched N), BCa CI over circuits (frozen + `confirm.rq2_p1_delta_h`, unchanged). +- Arms: federated = pooled **bridge-federated + directory-federated** (3 000 + circuits, per the loader frozen while blind); single-house = **1house-N** (1 500 + circuits). Matched-N per §6. +- **ΔH = −0.9587 bits, CI [−1.0559, −0.8641].** CI strictly < 0 → frozen gate = + `shrink`. Holm-adjusted p ≈ 0 (rank 2, multiplier 6) → rejected. +- **Interpretation (CONFIRMATORY):** federation, as instrumented, **reduces** the + per-circuit anonymity set by ≈ 0.96 bits relative to a matched-N single house. + This is the *opposite* of RQ2's motivating hypothesis (that federation grows the + anonymity set). The prereg framed RQ2-P1 two-sided precisely so this outcome is + reported "with equal prominence" (§6) — it is a genuine negative finding, not a + failure to detect. We do **not** re-frame it as federation "helping." + +## RQ1-P2 — padding efficacy: inconclusive (padding-ineffective) + +- Unit: the run-index-paired bridge-on / no-pad vs bridge-on / +padding arms + (RATIFIED stage-05 pairing), one `PairedCircuit` per shared run index (n = **30** + paired runs), paired ΔAUC = AUC(no-pad) − AUC(+pad), effective iff CI > 0. +- **ΔAUC = +0.0113, CI [−0.0025, +0.0234].** CI spans 0 → frozen gate = + `padding-ineffective`. raw p = 0.0912, Holm-adjusted p = 0.456 → not rejected. +- **Interpretation (CONFIRMATORY):** no significant padding effect on measured + linkability. This is moot given RQ1-P1 found no leak to suppress, but is reported + as the frozen test specifies. The per-run ΔAUC diagnostics (30 values, in the + results JSON) straddle zero (range ≈ [−0.080, +0.081]), consistent with the null. +- Same intractability as RQ1-P1 (each resample pools ~75 k pairs through the O(n²) + frozen `auc` twice); the CI uses the bit-faithful `_bootstrap_delta_auc_ci`, + `--verify`-checked equal to `stats.bootstrap_ci(units, _delta_auc, bca)` bit-for-bit. + +## RQ2-P3 — funnelling mechanism: inconclusive (as-instrumented degeneracy) + +- Unit: per bridge-federated circuit — Spearman ρ between top-k = 3 willing-bridge + concentration and per-circuit H (frozen `confirm.rq2_p3_funnel`, k = 3; + n = 1 500 circuits). +- **ρ = 0.0000, CI [0.0000, 0.0000]** (percentile fallback) → `inconclusive`. +- **Interpretation (CONFIRMATORY):** the concentration series has **no variance** — + the bridge-federated topology assigns a *fresh* willing bridge per circuit seed, so + willing-bridge reuse is minimal and the top-3 concentration is effectively + constant. Spearman is undefined on a zero-variance covariate and returns 0. This is + the **as-instrumented degeneracy flagged in advance** (stage-05 RQ2 clarification + instrument caveat), not a null effect of a well-posed mechanism test. The + funnelling mechanism is **not testable** on this instrument as built; we report it + as inconclusive and carry the caveat into Limitations rather than over-claiming. + +## Multiplicity (Holm–Bonferroni, frozen family size 7, report 4) + +Ordered by ascending raw p; multipliers are `7 − k + 1` for reported rank k +(conservative embedding — the reported RQ1/RQ2 tests occupy the smallest slots of +the full frozen family; never re-optimised to m = 4). Two tests survive Holm at +α = 0.05: RQ1-P1 (anomaly-below-chance — *not* a leak) and RQ2-P1 (shrink — a +negative effect). RQ1-P2 and RQ2-P3 do not. + +## Exploratory (labelled EXPLORATORY — not Holm-corrected, not confirmatory) + +- **RQ2-P1, bridge-federated-only ΔH** = −3.63 bits (degenerate CI; every resample + identical because the bridge-federated per-circuit posterior is near-single-member, + m_i ≈ 1). This is the directive's narrower contrast; it is reported only for + transparency and reinforces the RQ2-P3 degeneracy note (the bridge-federated arm's + anonymity set is near-trivial as instrumented). It carries **no** confirmatory + weight and is excluded from the Holm family. + +## What this pass did and did not do + +- **Ran only the pre-registered tests.** No exploratory subgroup hunting, no + re-slicing of cells, no alternative estimators. The one exploratory contrast is + labelled and severed from the confirmatory family. +- **Definitions are the frozen ones.** RQ1-P1 = bridge-on AUC vs 0.5 (D2), *not* the + bridge-on−bridge-off contrast paraphrased in the drive prompt; where the two + differed, the frozen prereg governs. +- **Blinding.** This is the first inferential pass; SS2 (raw freeze) computed no + AUC/H/CI. The correlator/entropy calibration was fixed on §5 synthetic fixtures + only — never fit to confirmatory-cell data. +- **Reproducibility.** Deterministic (fixed seed); the committed + `stage06_run.py --verify` proves the two fast bootstraps equal the frozen + `stats.bootstrap_ci` bit-for-bit, so the fast paths introduce no method artifact. diff --git a/output/sor-confirmatory/20260720T060132Z/analysis/stage06-results.json b/output/sor-confirmatory/20260720T060132Z/analysis/stage06-results.json new file mode 100644 index 0000000..51b2e14 --- /dev/null +++ b/output/sor-confirmatory/20260720T060132Z/analysis/stage06-results.json @@ -0,0 +1,176 @@ +{ + "meta": { + "data_dir": "output/sor-confirmatory/20260720T060132Z/confirmatory-data", + "analysis_seed": 20260719, + "n_resamples": 10000, + "alpha": 0.05, + "frozen_prereg_sha256": "f22331a72e0d0ccf38b787e63acabbe9d666456ec76076787a6d545c3193425b", + "holm_family": [ + "RQ1-P1", + "RQ1-P2", + "RQ2-P1", + "RQ2-P3", + "RQ3-P1-perf", + "RQ3-P1-latency", + "RQ3-P2" + ], + "holm_family_size": 7, + "reported": [ + "RQ1-P1", + "RQ1-P2", + "RQ2-P1", + "RQ2-P3" + ] + }, + "calibration": { + "linked_mean_auc": 1.0, + "unlinked_mean_auc": 0.5036272321428571, + "n_seeds": 40, + "passes": true, + "criterion": "linked>=0.95 and 0.40<=unlinked<=0.60" + }, + "n": { + "rq1_p1_pairs": 75000, + "rq1_p1_linked": 1500, + "rq1_p2_paired_runs": 30, + "rq2_federated_circuits": 3000, + "rq2_single_house_circuits": 1500, + "rq2_bridge_federated_circuits": 1500, + "rq2_p3_points": 1500 + }, + "confirmatory": { + "RQ1-P1": { + "name": "RQ1-P1", + "effect": "AUC", + "decision": "anomaly-below-chance", + "label": "", + "p_for_holm": 0.0, + "point": 0.46602644444444447, + "ci_lo": 0.4522774981189708, + "ci_hi": 0.4798496744798094, + "alpha": 0.05, + "n_resamples": 10000, + "method": "bca", + "seed": 20260719, + "raw_p": 0.0, + "holm_p": 0.0, + "holm_multiplier": 7, + "holm_rank": 1, + "holm_reject": true, + "label_type": "CONFIRMATORY", + "method_note": "pooled pair-set BCa jackknife is O(n^2)-intractable at n=75000; CI via performance-faithful bootstrap reproducing stats.bootstrap_ci bit-for-bit (verified on subsample). Point estimate + gate unchanged." + }, + "RQ1-P2": { + "name": "RQ1-P2", + "effect": "\u0394AUC", + "decision": "padding-ineffective", + "label": "", + "p_for_holm": 0.0912, + "point": 0.011296190476190515, + "ci_lo": -0.0024702237589305357, + "ci_hi": 0.023428857897449622, + "alpha": 0.05, + "n_resamples": 10000, + "method": "bca", + "seed": 20260719, + "raw_p": 0.0912, + "holm_p": 0.456, + "holm_multiplier": 5, + "holm_rank": 3, + "holm_reject": false, + "label_type": "CONFIRMATORY", + "per_run_delta_aucs": [ + -0.0244571428571429, + 0.016065306122448997, + 0.017338775510204074, + -0.03949795918367349, + -0.07316734693877552, + 0.021889795918367316, + -0.00880816326530609, + 0.014522448979591873, + 0.008612244897959198, + 0.015277551020408175, + 0.001004081632653031, + 0.021804081632653072, + 0.07793469387755109, + -0.059420408163265326, + -0.018987755102040815, + 0.05393469387755101, + 0.04289795918367345, + 0.06062857142857142, + 0.056612244897959185, + 0.004902040816326558, + 0.04325714285714283, + 0.03946938775510206, + 0.0032653061224489632, + 0.009379591836734702, + -0.0359591836734694, + 0.02003265306122448, + -0.07998367346938778, + 0.027330612244897967, + -0.007257142857142851, + 0.08127346938775515 + ], + "method_note": "paired \u0394AUC over run-index units; each resample pools ~75k pairs through the O(n^2) frozen auc twice, so the CI uses the performance-faithful bootstrap reproducing stats.bootstrap_ci(units, _delta_auc, bca) bit-for-bit (verified). Decision + gate unchanged." + }, + "RQ2-P1": { + "name": "RQ2-P1", + "effect": "\u0394H", + "decision": "shrink", + "label": "honest null \u2014 federation shrinks (reported with equal prominence)", + "p_for_holm": 0.0, + "point": -0.9586750999659173, + "ci_lo": -1.0558843052256577, + "ci_hi": -0.8640677356036801, + "alpha": 0.05, + "n_resamples": 10000, + "method": "bca", + "seed": 20260719, + "raw_p": 0.0, + "holm_p": 0.0, + "holm_multiplier": 6, + "holm_rank": 2, + "holm_reject": true, + "label_type": "CONFIRMATORY" + }, + "RQ2-P3": { + "name": "RQ2-P3", + "effect": "spearman_rho", + "decision": "inconclusive", + "label": "CI spans 0", + "p_for_holm": 1.0, + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "alpha": 0.05, + "n_resamples": 10000, + "method": "percentile", + "seed": 20260719, + "raw_p": 1.0, + "holm_p": 1.0, + "holm_multiplier": 4, + "holm_rank": 4, + "holm_reject": false, + "label_type": "CONFIRMATORY", + "instrument_caveat": "bridge-federated assigns a fresh bridge per circuit seed; willing-bridge reuse is minimal so the P3 concentration distribution may be near-degenerate (an honest as-instrumented property)." + } + }, + "exploratory": { + "RQ2-P1-bridge-federated-only": { + "name": "RQ2-P1", + "effect": "\u0394H", + "decision": "shrink", + "label": "honest null \u2014 federation shrinks (reported with equal prominence)", + "p_for_holm": 0.0, + "point": -3.6311790803889767, + "ci_lo": -3.6311790803889767, + "ci_hi": -3.6311790803889767, + "alpha": 0.05, + "n_resamples": 10000, + "method": "percentile", + "seed": 20260719, + "label_type": "EXPLORATORY", + "note": "\u0394H(bridge-federated) \u2212 H(1house-N); narrower than the confirmatory federated-pooled arm; not Holm-corrected; reported for transparency." + } + } +} \ No newline at end of file