"""RQ3 companion executor-wiring — SYNTHETIC / OFFLINE only (`rq3-companion-run-brief.md` §2-4). No confirmatory battery is run and no confirmatory record is read; these pins exercise the build + the calibration gate. Pins: (a) the RQ3 cells + interleaved schedule are well-formed and kept DISJOINT from the frozen lead lattice (which stays byte-identical); (b) the offline selector DVs (throughput-retention, drops/rebuilds, rebuild-interval gaps) are deterministic and never fabricate a live-only added-latency; (c) the confirmatory `run_rq3_battery` refuses to run offline (its added-latency DV is a real measurement); (d) the agent arm replays byte-identically from its committed decision cache (no network); (e) the four §3-4 calibration gate items pass green with real teeth. """ from pathlib import Path import pytest from cmd_chat.sor.agent_selector import OllamaAgentPolicy from cmd_chat.sor.analysis.rq3_calibration import calibration_gate from cmd_chat.sor.battery import ( RQ3_CHURN_STEPS, RQ3_KILL_PROB_PCT, derive_seed, enumerate_cells, enumerate_rq3_cells, rq3_schedule, ) from cmd_chat.sor.churn import churn_schedule from cmd_chat.sor.executor import ( ExecutorError, rebuild_interval_gaps, run_rq3_battery, run_rq3_cell_run, ) from cmd_chat.sor.selector import Rebuild, SelectionResult, run_selection S0 = 20260719 # --- cells + schedule -------------------------------------------------------- # def test_rq3_cells_shape_and_pinned_churn(): cells = enumerate_rq3_cells() assert [c.factors["selector"] for c in cells] == ["static", "random", "agent"] assert all(c.rq == "RQ3" for c in cells) assert [c.is_control for c in cells] == [True, False, False] # static is baseline for c in cells: assert c.factors["topology"] == "1house" and c.factors["bridge"] == "off" assert int(c.factors["churn_kill_prob_pct"]) == RQ3_KILL_PROB_PCT == 30 assert int(c.factors["churn_steps"]) == RQ3_CHURN_STEPS == 20 def test_rq3_cells_disjoint_from_frozen_lead_lattice(): lead = enumerate_cells() assert len(lead) == 6 # the frozen lead lattice is untouched lead_ids = {c.cell_id for c in lead} assert lead_ids.isdisjoint({c.cell_id for c in enumerate_rq3_cells()}) def test_rq3_schedule_brackets_control_and_is_deterministic(): sched = rq3_schedule(S0, r=6) assert len(sched) == 3 * 6 control_id = next(c.cell_id for c in enumerate_rq3_cells() if c.is_control) is_ctrl = [pr.cell_id == control_id for pr in sched] # control interleaved BEFORE and AFTER the {random, agent} treatment block assert is_ctrl[0] and is_ctrl[-1] first_treat = is_ctrl.index(False) last_treat = len(is_ctrl) - 1 - is_ctrl[::-1].index(False) assert not any(is_ctrl[first_treat:last_treat + 1]) # treatments contiguous assert rq3_schedule(S0, r=6) == sched # same order_seed -> identical order # --- rebuild-interval-gap signal -------------------------------------------- # def test_rebuild_interval_gaps_sorted_diffs_and_null(): res = SelectionResult("static", 1, 3, ["a", "b", "c"], rebuilds=[ Rebuild(5, "x", []), Rebuild(2, "y", []), Rebuild(2, "z", []), Rebuild(9, "w", []), ]) assert rebuild_interval_gaps(res) == [0.0, 3.0, 4.0] # sorted [2,2,5,9] -> diffs lone = SelectionResult("static", 1, 3, ["a", "b", "c"], rebuilds=[Rebuild(4, "x", [])]) assert rebuild_interval_gaps(lone) == [] # <2 rebuilds -> no interval (not fabricated) # --- offline selector DVs ---------------------------------------------------- # def test_run_rq3_cell_run_offline_dvs_and_no_fabricated_latency(tmp_path): cell = next(c for c in enumerate_rq3_cells() if c.factors["selector"] == "agent") rep = run_rq3_cell_run(cell, 0, tmp_path, live=False) assert rep.added_latency_ms is None # never fabricated in the offline path assert 0.0 <= rep.throughput_retention <= 1.0 assert rep.kill_prob_pct == 30 and rep.steps == 20 assert rep.drops > 0 and rep.rebuilds > 0 # the pinned churn genuinely bites assert (Path(rep.run_dir) / "rq3-run.json").exists() # write-once DV sidecar def test_run_rq3_cell_run_is_deterministic(tmp_path): cell = next(c for c in enumerate_rq3_cells() if c.factors["selector"] == "random") a = run_rq3_cell_run(cell, 3, tmp_path / "a", live=False) b = run_rq3_cell_run(cell, 3, tmp_path / "b", live=False) assert (a.seed, a.drops, a.rebuilds, a.deferred) == (b.seed, b.drops, b.rebuilds, b.deferred) assert a.throughput_retention == b.throughput_retention assert a.rebuild_gaps == b.rebuild_gaps def test_run_rq3_battery_refuses_offline(tmp_path): with pytest.raises(ExecutorError): run_rq3_battery(tmp_path, r_runs=2, c_circuits=1, live=False) # --- agent arm: committed-cache replay is byte-identical (no network) -------- # def test_agent_cache_replay_is_reproducible(): nodes = [f"1house/node{i:02d}" for i in range(8)] seed = derive_seed("rq3-cache-replay", 0) sched = churn_schedule(seed, nodes, 20, kill_prob_pct=30) # host is deliberately unreachable → model query fails → deterministic heuristic # fallback; the decisions are cached as they are made. pol1 = OllamaAgentPolicy(host="http://127.0.0.1:9", timeout=0.05) first = run_selection(seed, nodes, 3, sched, policy=pol1) cache = dict(pol1._cache) assert cache # decisions were recorded # replay from the committed cache alone → identical circuits, all cache-sourced. pol2 = OllamaAgentPolicy(host="http://127.0.0.1:9", timeout=0.05, cache=cache) second = run_selection(seed, nodes, 3, sched, policy=pol2) assert second.initial_circuit == first.initial_circuit assert [rb.circuit for rb in second.rebuilds] == [rb.circuit for rb in first.rebuilds] assert all(d["source"] == "cache" for d in pol2.decisions()) # --- calibration gate has real teeth (§3-4) ---------------------------------- # def test_rq3_calibration_gate_all_items_pass_with_teeth(): report = calibration_gate() gate = report["gate"] assert gate["all_pass"] is True bites = gate["item1_churn_bites"] assert bites["total_drops"] > 0 and bites["total_rebuilds"] > 0 clf = gate["item2_rebuild_classifier"] assert clf["separable_churned_vs_baseline"] and clf["null_baseline_vs_baseline"] assert clf["auc_churned_vs_baseline"] >= 0.90 # regime signal has teeth assert abs(clf["auc_baseline_vs_baseline"] - 0.5) <= 0.15 # same-regime null is blind assert gate["item3_agent_reproducibility"]["pass"] assert gate["item4_entropy_calibration"]["pass"]