R7/RQ3: executor-wiring stone — offline selector DVs + calibration gate
Wire the RQ3 churn-resilience companion (severable follow-on; RQ3 frozen in its
own prereg) as SYNTHETIC/OFFLINE build + calibration only — no confirmatory
battery run.
- battery: enumerate_rq3_cells()/rq3_schedule() add selector in {static(control),
random, agent} at 1house/bridge-off under pinned churn kp=30/steps=20, kept
SEPARATE so the frozen 6-cell lead lattice stays byte-identical.
- executor: run_rq3_cell_run collects offline selector DVs (throughput-retention,
drops/rebuilds, rebuild-interval gaps); added-latency is live-only (offline path
records None, never fabricated); run_rq3_battery(live=False) hard-raises.
- analysis/rq3_calibration: DRY offline gate. Churn-bites (1589 drops/1576
rebuilds) + rebuild-classifier calibration green with real teeth — churned(kp30)
vs baseline(kp5) AUC 0.926 separable, baseline-vs-baseline null 0.518 blind.
Classifier scored on the PER-RUN mean inter-rebuild gap (the confirmatory
grouping unit); the frozen instrument (rebuild_interval_gaps,
rebuild_classifier_auc) is untouched, no fit to confirmatory data.
HARD HOLD: no RQ3 confirmatory battery (live added-latency is operator+grid-gated).
Lead prereg SHA f22331a72e… untouched; containment intact (synthetic/offline, $0
local Ollama, frontier arm inert); worktree-only.
Tests: tests/test_sor_rq3_wiring.py 9 passed; full SOR suite 194 passed.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,139 @@
|
||||
"""RQ3 companion executor-wiring — SYNTHETIC / OFFLINE only
|
||||
(`rq3-companion-run-brief.md` §2-4). No confirmatory battery is run and no
|
||||
confirmatory record is read; these pins exercise the build + the calibration gate.
|
||||
|
||||
Pins: (a) the RQ3 cells + interleaved schedule are well-formed and kept DISJOINT
|
||||
from the frozen lead lattice (which stays byte-identical); (b) the offline selector
|
||||
DVs (throughput-retention, drops/rebuilds, rebuild-interval gaps) are deterministic
|
||||
and never fabricate a live-only added-latency; (c) the confirmatory `run_rq3_battery`
|
||||
refuses to run offline (its added-latency DV is a real measurement); (d) the agent
|
||||
arm replays byte-identically from its committed decision cache (no network); (e) the
|
||||
four §3-4 calibration gate items pass green with real teeth.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from cmd_chat.sor.agent_selector import OllamaAgentPolicy
|
||||
from cmd_chat.sor.analysis.rq3_calibration import calibration_gate
|
||||
from cmd_chat.sor.battery import (
|
||||
RQ3_CHURN_STEPS,
|
||||
RQ3_KILL_PROB_PCT,
|
||||
derive_seed,
|
||||
enumerate_cells,
|
||||
enumerate_rq3_cells,
|
||||
rq3_schedule,
|
||||
)
|
||||
from cmd_chat.sor.churn import churn_schedule
|
||||
from cmd_chat.sor.executor import (
|
||||
ExecutorError,
|
||||
rebuild_interval_gaps,
|
||||
run_rq3_battery,
|
||||
run_rq3_cell_run,
|
||||
)
|
||||
from cmd_chat.sor.selector import Rebuild, SelectionResult, run_selection
|
||||
|
||||
S0 = 20260719
|
||||
|
||||
|
||||
# --- cells + schedule -------------------------------------------------------- #
|
||||
def test_rq3_cells_shape_and_pinned_churn():
|
||||
cells = enumerate_rq3_cells()
|
||||
assert [c.factors["selector"] for c in cells] == ["static", "random", "agent"]
|
||||
assert all(c.rq == "RQ3" for c in cells)
|
||||
assert [c.is_control for c in cells] == [True, False, False] # static is baseline
|
||||
for c in cells:
|
||||
assert c.factors["topology"] == "1house" and c.factors["bridge"] == "off"
|
||||
assert int(c.factors["churn_kill_prob_pct"]) == RQ3_KILL_PROB_PCT == 30
|
||||
assert int(c.factors["churn_steps"]) == RQ3_CHURN_STEPS == 20
|
||||
|
||||
|
||||
def test_rq3_cells_disjoint_from_frozen_lead_lattice():
|
||||
lead = enumerate_cells()
|
||||
assert len(lead) == 6 # the frozen lead lattice is untouched
|
||||
lead_ids = {c.cell_id for c in lead}
|
||||
assert lead_ids.isdisjoint({c.cell_id for c in enumerate_rq3_cells()})
|
||||
|
||||
|
||||
def test_rq3_schedule_brackets_control_and_is_deterministic():
|
||||
sched = rq3_schedule(S0, r=6)
|
||||
assert len(sched) == 3 * 6
|
||||
control_id = next(c.cell_id for c in enumerate_rq3_cells() if c.is_control)
|
||||
is_ctrl = [pr.cell_id == control_id for pr in sched]
|
||||
# control interleaved BEFORE and AFTER the {random, agent} treatment block
|
||||
assert is_ctrl[0] and is_ctrl[-1]
|
||||
first_treat = is_ctrl.index(False)
|
||||
last_treat = len(is_ctrl) - 1 - is_ctrl[::-1].index(False)
|
||||
assert not any(is_ctrl[first_treat:last_treat + 1]) # treatments contiguous
|
||||
assert rq3_schedule(S0, r=6) == sched # same order_seed -> identical order
|
||||
|
||||
|
||||
# --- rebuild-interval-gap signal -------------------------------------------- #
|
||||
def test_rebuild_interval_gaps_sorted_diffs_and_null():
|
||||
res = SelectionResult("static", 1, 3, ["a", "b", "c"], rebuilds=[
|
||||
Rebuild(5, "x", []), Rebuild(2, "y", []), Rebuild(2, "z", []), Rebuild(9, "w", []),
|
||||
])
|
||||
assert rebuild_interval_gaps(res) == [0.0, 3.0, 4.0] # sorted [2,2,5,9] -> diffs
|
||||
lone = SelectionResult("static", 1, 3, ["a", "b", "c"], rebuilds=[Rebuild(4, "x", [])])
|
||||
assert rebuild_interval_gaps(lone) == [] # <2 rebuilds -> no interval (not fabricated)
|
||||
|
||||
|
||||
# --- offline selector DVs ---------------------------------------------------- #
|
||||
def test_run_rq3_cell_run_offline_dvs_and_no_fabricated_latency(tmp_path):
|
||||
cell = next(c for c in enumerate_rq3_cells() if c.factors["selector"] == "agent")
|
||||
rep = run_rq3_cell_run(cell, 0, tmp_path, live=False)
|
||||
assert rep.added_latency_ms is None # never fabricated in the offline path
|
||||
assert 0.0 <= rep.throughput_retention <= 1.0
|
||||
assert rep.kill_prob_pct == 30 and rep.steps == 20
|
||||
assert rep.drops > 0 and rep.rebuilds > 0 # the pinned churn genuinely bites
|
||||
assert (Path(rep.run_dir) / "rq3-run.json").exists() # write-once DV sidecar
|
||||
|
||||
|
||||
def test_run_rq3_cell_run_is_deterministic(tmp_path):
|
||||
cell = next(c for c in enumerate_rq3_cells() if c.factors["selector"] == "random")
|
||||
a = run_rq3_cell_run(cell, 3, tmp_path / "a", live=False)
|
||||
b = run_rq3_cell_run(cell, 3, tmp_path / "b", live=False)
|
||||
assert (a.seed, a.drops, a.rebuilds, a.deferred) == (b.seed, b.drops, b.rebuilds, b.deferred)
|
||||
assert a.throughput_retention == b.throughput_retention
|
||||
assert a.rebuild_gaps == b.rebuild_gaps
|
||||
|
||||
|
||||
def test_run_rq3_battery_refuses_offline(tmp_path):
|
||||
with pytest.raises(ExecutorError):
|
||||
run_rq3_battery(tmp_path, r_runs=2, c_circuits=1, live=False)
|
||||
|
||||
|
||||
# --- agent arm: committed-cache replay is byte-identical (no network) -------- #
|
||||
def test_agent_cache_replay_is_reproducible():
|
||||
nodes = [f"1house/node{i:02d}" for i in range(8)]
|
||||
seed = derive_seed("rq3-cache-replay", 0)
|
||||
sched = churn_schedule(seed, nodes, 20, kill_prob_pct=30)
|
||||
# host is deliberately unreachable → model query fails → deterministic heuristic
|
||||
# fallback; the decisions are cached as they are made.
|
||||
pol1 = OllamaAgentPolicy(host="http://127.0.0.1:9", timeout=0.05)
|
||||
first = run_selection(seed, nodes, 3, sched, policy=pol1)
|
||||
cache = dict(pol1._cache)
|
||||
assert cache # decisions were recorded
|
||||
|
||||
# replay from the committed cache alone → identical circuits, all cache-sourced.
|
||||
pol2 = OllamaAgentPolicy(host="http://127.0.0.1:9", timeout=0.05, cache=cache)
|
||||
second = run_selection(seed, nodes, 3, sched, policy=pol2)
|
||||
assert second.initial_circuit == first.initial_circuit
|
||||
assert [rb.circuit for rb in second.rebuilds] == [rb.circuit for rb in first.rebuilds]
|
||||
assert all(d["source"] == "cache" for d in pol2.decisions())
|
||||
|
||||
|
||||
# --- calibration gate has real teeth (§3-4) ---------------------------------- #
|
||||
def test_rq3_calibration_gate_all_items_pass_with_teeth():
|
||||
report = calibration_gate()
|
||||
gate = report["gate"]
|
||||
assert gate["all_pass"] is True
|
||||
bites = gate["item1_churn_bites"]
|
||||
assert bites["total_drops"] > 0 and bites["total_rebuilds"] > 0
|
||||
clf = gate["item2_rebuild_classifier"]
|
||||
assert clf["separable_churned_vs_baseline"] and clf["null_baseline_vs_baseline"]
|
||||
assert clf["auc_churned_vs_baseline"] >= 0.90 # regime signal has teeth
|
||||
assert abs(clf["auc_baseline_vs_baseline"] - 0.5) <= 0.15 # same-regime null is blind
|
||||
assert gate["item3_agent_reproducibility"]["pass"]
|
||||
assert gate["item4_entropy_calibration"]["pass"]
|
||||
Reference in New Issue
Block a user