SS3: frozen §6 confirmatory pass on real 180/180 — honest null/negative
One auditable, deterministic run of the four lead-paper §6 tests on the frozen raw battery. Calibration gate PASS (linked AUC 1.0000, unlinked 0.5036 on §5 fixtures) → AUCs reportable. - RQ1-P1 (bridge leak): AUC 0.4660 CI[0.4523,0.4798] = anomaly-below-chance. NO measurable leak; Holm-sig but in the WRONG direction (not evidence of linkability). - RQ2-P1 (federation): ΔH −0.9587 bits CI[−1.056,−0.864] = SHRINK. Holm-sig HONEST-NEGATIVE — federation reduces the anonymity set (opposite of RQ2's motivation); reported with equal prominence per §6 two-sided framing. - RQ1-P2 (padding): ΔAUC +0.0113 CI[−0.0025,+0.0234] padding-ineffective (raw p 0.091, Holm adj-p 0.456, not rejected). - RQ2-P3 (funnelling): Spearman ρ=0, zero-variance concentration = as-instrumented degeneracy flagged in advance; inconclusive. Holm family=7 report-4 (multipliers 7,6,5,4); only RQ1-P1 + RQ2-P1 survive. Ran ONLY the pre-registered tests — no re-slicing, no post-hoc subgroups. Method faithfulness: RQ1-P1/P2 CIs use a bit-faithful fast numpy bootstrap (frozen O(n²)-per-fold jackknife intractable at n=75000); stage06_run.py --verify proves both == frozen stats.bootstrap_ci bit-for-bit (point/lo/hi/ method within 1e-12, incl the average-rank tie path). RQ2-P1/P3 stay on the frozen confirm.* paths (tractable). Frozen prereg SHA INTACT. Artifacts: docs/stage-06-analysis.md (methods-faithful narrative, Holm table, CONFIRMATORY/EXPLORATORY labels, nulls reported honestly) + force-added output/…/analysis/stage06-results.json (regenerable via stage06_run). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
leetcrypt
@@ -0,0 +1,176 @@
|
||||
{
|
||||
"meta": {
|
||||
"data_dir": "output/sor-confirmatory/20260720T060132Z/confirmatory-data",
|
||||
"analysis_seed": 20260719,
|
||||
"n_resamples": 10000,
|
||||
"alpha": 0.05,
|
||||
"frozen_prereg_sha256": "f22331a72e0d0ccf38b787e63acabbe9d666456ec76076787a6d545c3193425b",
|
||||
"holm_family": [
|
||||
"RQ1-P1",
|
||||
"RQ1-P2",
|
||||
"RQ2-P1",
|
||||
"RQ2-P3",
|
||||
"RQ3-P1-perf",
|
||||
"RQ3-P1-latency",
|
||||
"RQ3-P2"
|
||||
],
|
||||
"holm_family_size": 7,
|
||||
"reported": [
|
||||
"RQ1-P1",
|
||||
"RQ1-P2",
|
||||
"RQ2-P1",
|
||||
"RQ2-P3"
|
||||
]
|
||||
},
|
||||
"calibration": {
|
||||
"linked_mean_auc": 1.0,
|
||||
"unlinked_mean_auc": 0.5036272321428571,
|
||||
"n_seeds": 40,
|
||||
"passes": true,
|
||||
"criterion": "linked>=0.95 and 0.40<=unlinked<=0.60"
|
||||
},
|
||||
"n": {
|
||||
"rq1_p1_pairs": 75000,
|
||||
"rq1_p1_linked": 1500,
|
||||
"rq1_p2_paired_runs": 30,
|
||||
"rq2_federated_circuits": 3000,
|
||||
"rq2_single_house_circuits": 1500,
|
||||
"rq2_bridge_federated_circuits": 1500,
|
||||
"rq2_p3_points": 1500
|
||||
},
|
||||
"confirmatory": {
|
||||
"RQ1-P1": {
|
||||
"name": "RQ1-P1",
|
||||
"effect": "AUC",
|
||||
"decision": "anomaly-below-chance",
|
||||
"label": "",
|
||||
"p_for_holm": 0.0,
|
||||
"point": 0.46602644444444447,
|
||||
"ci_lo": 0.4522774981189708,
|
||||
"ci_hi": 0.4798496744798094,
|
||||
"alpha": 0.05,
|
||||
"n_resamples": 10000,
|
||||
"method": "bca",
|
||||
"seed": 20260719,
|
||||
"raw_p": 0.0,
|
||||
"holm_p": 0.0,
|
||||
"holm_multiplier": 7,
|
||||
"holm_rank": 1,
|
||||
"holm_reject": true,
|
||||
"label_type": "CONFIRMATORY",
|
||||
"method_note": "pooled pair-set BCa jackknife is O(n^2)-intractable at n=75000; CI via performance-faithful bootstrap reproducing stats.bootstrap_ci bit-for-bit (verified on subsample). Point estimate + gate unchanged."
|
||||
},
|
||||
"RQ1-P2": {
|
||||
"name": "RQ1-P2",
|
||||
"effect": "\u0394AUC",
|
||||
"decision": "padding-ineffective",
|
||||
"label": "",
|
||||
"p_for_holm": 0.0912,
|
||||
"point": 0.011296190476190515,
|
||||
"ci_lo": -0.0024702237589305357,
|
||||
"ci_hi": 0.023428857897449622,
|
||||
"alpha": 0.05,
|
||||
"n_resamples": 10000,
|
||||
"method": "bca",
|
||||
"seed": 20260719,
|
||||
"raw_p": 0.0912,
|
||||
"holm_p": 0.456,
|
||||
"holm_multiplier": 5,
|
||||
"holm_rank": 3,
|
||||
"holm_reject": false,
|
||||
"label_type": "CONFIRMATORY",
|
||||
"per_run_delta_aucs": [
|
||||
-0.0244571428571429,
|
||||
0.016065306122448997,
|
||||
0.017338775510204074,
|
||||
-0.03949795918367349,
|
||||
-0.07316734693877552,
|
||||
0.021889795918367316,
|
||||
-0.00880816326530609,
|
||||
0.014522448979591873,
|
||||
0.008612244897959198,
|
||||
0.015277551020408175,
|
||||
0.001004081632653031,
|
||||
0.021804081632653072,
|
||||
0.07793469387755109,
|
||||
-0.059420408163265326,
|
||||
-0.018987755102040815,
|
||||
0.05393469387755101,
|
||||
0.04289795918367345,
|
||||
0.06062857142857142,
|
||||
0.056612244897959185,
|
||||
0.004902040816326558,
|
||||
0.04325714285714283,
|
||||
0.03946938775510206,
|
||||
0.0032653061224489632,
|
||||
0.009379591836734702,
|
||||
-0.0359591836734694,
|
||||
0.02003265306122448,
|
||||
-0.07998367346938778,
|
||||
0.027330612244897967,
|
||||
-0.007257142857142851,
|
||||
0.08127346938775515
|
||||
],
|
||||
"method_note": "paired \u0394AUC over run-index units; each resample pools ~75k pairs through the O(n^2) frozen auc twice, so the CI uses the performance-faithful bootstrap reproducing stats.bootstrap_ci(units, _delta_auc, bca) bit-for-bit (verified). Decision + gate unchanged."
|
||||
},
|
||||
"RQ2-P1": {
|
||||
"name": "RQ2-P1",
|
||||
"effect": "\u0394H",
|
||||
"decision": "shrink",
|
||||
"label": "honest null \u2014 federation shrinks (reported with equal prominence)",
|
||||
"p_for_holm": 0.0,
|
||||
"point": -0.9586750999659173,
|
||||
"ci_lo": -1.0558843052256577,
|
||||
"ci_hi": -0.8640677356036801,
|
||||
"alpha": 0.05,
|
||||
"n_resamples": 10000,
|
||||
"method": "bca",
|
||||
"seed": 20260719,
|
||||
"raw_p": 0.0,
|
||||
"holm_p": 0.0,
|
||||
"holm_multiplier": 6,
|
||||
"holm_rank": 2,
|
||||
"holm_reject": true,
|
||||
"label_type": "CONFIRMATORY"
|
||||
},
|
||||
"RQ2-P3": {
|
||||
"name": "RQ2-P3",
|
||||
"effect": "spearman_rho",
|
||||
"decision": "inconclusive",
|
||||
"label": "CI spans 0",
|
||||
"p_for_holm": 1.0,
|
||||
"point": 0.0,
|
||||
"ci_lo": 0.0,
|
||||
"ci_hi": 0.0,
|
||||
"alpha": 0.05,
|
||||
"n_resamples": 10000,
|
||||
"method": "percentile",
|
||||
"seed": 20260719,
|
||||
"raw_p": 1.0,
|
||||
"holm_p": 1.0,
|
||||
"holm_multiplier": 4,
|
||||
"holm_rank": 4,
|
||||
"holm_reject": false,
|
||||
"label_type": "CONFIRMATORY",
|
||||
"instrument_caveat": "bridge-federated assigns a fresh bridge per circuit seed; willing-bridge reuse is minimal so the P3 concentration distribution may be near-degenerate (an honest as-instrumented property)."
|
||||
}
|
||||
},
|
||||
"exploratory": {
|
||||
"RQ2-P1-bridge-federated-only": {
|
||||
"name": "RQ2-P1",
|
||||
"effect": "\u0394H",
|
||||
"decision": "shrink",
|
||||
"label": "honest null \u2014 federation shrinks (reported with equal prominence)",
|
||||
"p_for_holm": 0.0,
|
||||
"point": -3.6311790803889767,
|
||||
"ci_lo": -3.6311790803889767,
|
||||
"ci_hi": -3.6311790803889767,
|
||||
"alpha": 0.05,
|
||||
"n_resamples": 10000,
|
||||
"method": "percentile",
|
||||
"seed": 20260719,
|
||||
"label_type": "EXPLORATORY",
|
||||
"note": "\u0394H(bridge-federated) \u2212 H(1house-N); narrower than the confirmatory federated-pooled arm; not Holm-corrected; reported for transparency."
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user